__ __ / \ / \ ______ _ _ __ __ __ / /\ / /\ \ / __ \ | | | | / \ \ \ / / / / / / \ \ | | | | | | | | / /\ \ \ / \ \ \ \ / / | |__| | | |__| | / ____ \ | | \ \/ \ \/ / \_ ___/ \____/ /_/ \_\ |_| \__/ \__/ \ \__ \___\ by Red Hat Build, Store, and Distribute your Containers Startup timestamp: Tue Nov 4 09:03:10 AM UTC 2025 Running all default registry services without migration Running init script '/quay-registry/conf/init/certs_install.sh' Installing extra certificates found in /quay-registry/conf/stack/extra_ca_certs directory Running init script '/quay-registry/conf/init/client_certs.sh' Running init script '/quay-registry/conf/init/copy_config_files.sh' Running init script '/quay-registry/conf/init/d_validate_config_bundle.sh' Validating Configuration time="2025-11-04T09:03:11Z" level=debug msg="Validating AccessSettings" time="2025-11-04T09:03:11Z" level=debug msg="Validating ActionLogArchiving" time="2025-11-04T09:03:11Z" level=debug msg="Validating AppTokenAuthentication" time="2025-11-04T09:03:11Z" level=debug msg="Validating AutoPrune" time="2025-11-04T09:03:11Z" level=debug msg="Validating BitbucketBuildTrigger" time="2025-11-04T09:03:11Z" level=debug msg="Validating BuildManager" time="2025-11-04T09:03:11Z" level=debug msg="Validating Database" time="2025-11-04T09:03:11Z" level=debug msg="Scheme: postgresql" time="2025-11-04T09:03:11Z" level=debug msg="Host: quayregistry-quay-database:5432" time="2025-11-04T09:03:11Z" level=debug msg="Db: quayregistry-quay-database" time="2025-11-04T09:03:11Z" level=debug msg="Params: " time="2025-11-04T09:03:11Z" level=debug msg="Including params " time="2025-11-04T09:03:11Z" level=debug msg="Pinging database at hostname: quayregistry-quay-database:5432." time="2025-11-04T09:03:11Z" level=debug msg="Database version: 13.20" plpgsql pg_trgm time="2025-11-04T09:03:11Z" level=debug msg="Validating DistributedStorage" time="2025-11-04T09:03:11Z" level=debug msg="Using IBM Cloud/ODF/RadosGW storage." time="2025-11-04T09:03:11Z" level=debug msg="Storage parameters: " time="2025-11-04T09:03:11Z" level=debug msg="hostname: s3.openshift-storage.svc.cluster.local:443, bucket name: quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84, TLS enabled: true" time="2025-11-04T09:03:11Z" level=debug msg="Validating ElasticSearch" time="2025-11-04T09:03:11Z" level=debug msg="Validating Email" time="2025-11-04T09:03:11Z" level=debug msg="Validating GitHubBuildTrigger" time="2025-11-04T09:03:11Z" level=debug msg="Validating GitHubLogin" time="2025-11-04T09:03:11Z" level=debug msg="Validating GitLabBuildTrigger" time="2025-11-04T09:03:11Z" level=debug msg="Validating GoogleLogin" time="2025-11-04T09:03:11Z" level=debug msg="Validating HostSettings" time="2025-11-04T09:03:11Z" level=debug msg="Validating JWTAuthentication" time="2025-11-04T09:03:11Z" level=debug msg="Validating LDAP" time="2025-11-04T09:03:11Z" level=debug msg="Validating OIDC" time="2025-11-04T09:03:11Z" level=debug msg="Validating QuayDocumentation" time="2025-11-04T09:03:11Z" level=debug msg="Validating Redis" time="2025-11-04T09:03:11Z" level=debug msg="Address: quayregistry-quay-redis:6379" time="2025-11-04T09:03:11Z" level=debug msg="Username: " time="2025-11-04T09:03:11Z" level=debug msg="Password Len: 0" time="2025-11-04T09:03:11Z" level=debug msg="Ssl: " time="2025-11-04T09:03:11Z" level=debug msg="Address: quayregistry-quay-redis:6379" time="2025-11-04T09:03:11Z" level=debug msg="Username: " time="2025-11-04T09:03:11Z" level=debug msg="Password Len: 0" time="2025-11-04T09:03:11Z" level=debug msg="Ssl: " time="2025-11-04T09:03:11Z" level=debug msg="Address: quayregistry-quay-redis:6379" time="2025-11-04T09:03:11Z" level=debug msg="Username: " time="2025-11-04T09:03:11Z" level=debug msg="Password Len: 0" time="2025-11-04T09:03:11Z" level=debug msg="Ssl: " time="2025-11-04T09:03:11Z" level=debug msg="Validating RepoMirror" time="2025-11-04T09:03:11Z" level=debug msg="Validating SecurityScanner" time="2025-11-04T09:03:11Z" level=debug msg="Validating TeamSyncing" time="2025-11-04T09:03:11Z" level=debug msg="Validating TimeMachine" time="2025-11-04T09:03:11Z" level=debug msg="Validating UserVisibleSettings" +------------------------+-------+--------+ | Field Group | Error | Status | +------------------------+-------+--------+ | AccessSettings | - | 🟢 | +------------------------+-------+--------+ | ActionLogArchiving | - | 🟢 | +------------------------+-------+--------+ | AppTokenAuthentication | - | 🟢 | +------------------------+-------+--------+ | AutoPrune | - | 🟢 | +------------------------+-------+--------+ | BitbucketBuildTrigger | - | 🟢 | +------------------------+-------+--------+ | BuildManager | - | 🟢 | +------------------------+-------+--------+ | Database | - | 🟢 | +------------------------+-------+--------+ | DistributedStorage | - | 🟢 | +------------------------+-------+--------+ | ElasticSearch | - | 🟢 | +------------------------+-------+--------+ | Email | - | 🟢 | +------------------------+-------+--------+ | GitHubBuildTrigger | - | 🟢 | +------------------------+-------+--------+ | GitHubLogin | - | 🟢 | +------------------------+-------+--------+ | GitLabBuildTrigger | - | 🟢 | +------------------------+-------+--------+ | GoogleLogin | - | 🟢 | +------------------------+-------+--------+ | HostSettings | - | 🟢 | +------------------------+-------+--------+ | JWTAuthentication | - | 🟢 | +------------------------+-------+--------+ | LDAP | - | 🟢 | +------------------------+-------+--------+ | OIDC | - | 🟢 | +------------------------+-------+--------+ | QuayDocumentation | - | 🟢 | +------------------------+-------+--------+ | Redis | - | 🟢 | +------------------------+-------+--------+ | RepoMirror | - | 🟢 | +------------------------+-------+--------+ | SecurityScanner | - | 🟢 | +------------------------+-------+--------+ | TeamSyncing | - | 🟢 | +------------------------+-------+--------+ | TimeMachine | - | 🟢 | +------------------------+-------+--------+ | UserVisibleSettings | - | 🟢 | +------------------------+-------+--------+ Running init script '/quay-registry/conf/init/nginx_conf_create.sh' Running init script '/quay-registry/conf/init/supervisord_conf_create.sh' Running init script '/quay-registry/conf/init/zz_boot.sh' fatal: not a git repository: /quay-registry/../.git/modules/quay /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' return re.sub('[^\w]+', '_', self.name) /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) /quay-registry/boot.py:76: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). expiration = datetime.utcnow() + timedelta(minutes=minutes_until_expiration) 2025-11-04 09:03:16,350 INFO RPC interface 'supervisor' initialized 2025-11-04 09:03:16,350 CRIT Server 'unix_http_server' running without any HTTP authentication checking 2025-11-04 09:03:16,350 INFO supervisord started with pid 2 2025-11-04 09:03:17,352 INFO spawned: 'stdout' with pid 57 2025-11-04 09:03:17,355 INFO spawned: 'autopruneworker' with pid 58 2025-11-04 09:03:17,356 INFO spawned: 'blobuploadcleanupworker' with pid 59 2025-11-04 09:03:17,358 INFO spawned: 'builder' with pid 60 2025-11-04 09:03:17,360 INFO spawned: 'buildlogsarchiver' with pid 61 2025-11-04 09:03:17,362 INFO spawned: 'chunkcleanupworker' with pid 62 2025-11-04 09:03:17,364 INFO spawned: 'dnsmasq' with pid 63 2025-11-04 09:03:17,366 INFO spawned: 'expiredappspecifictokenworker' with pid 64 2025-11-04 09:03:17,367 INFO spawned: 'exportactionlogsworker' with pid 65 2025-11-04 09:03:17,370 INFO spawned: 'gcworker' with pid 66 2025-11-04 09:03:17,372 INFO spawned: 'globalpromstats' with pid 67 2025-11-04 09:03:17,374 INFO spawned: 'gunicorn-registry' with pid 68 2025-11-04 09:03:17,377 INFO spawned: 'gunicorn-secscan' with pid 69 2025-11-04 09:03:17,379 INFO spawned: 'gunicorn-web' with pid 70 2025-11-04 09:03:17,381 INFO spawned: 'logrotateworker' with pid 71 2025-11-04 09:03:17,383 INFO spawned: 'manifestbackfillworker' with pid 72 2025-11-04 09:03:17,385 INFO spawned: 'manifestsubjectbackfillworker' with pid 73 2025-11-04 09:03:17,460 INFO spawned: 'memcache' with pid 74 2025-11-04 09:03:17,462 INFO spawned: 'namespacegcworker' with pid 75 2025-11-04 09:03:17,464 INFO spawned: 'nginx' with pid 76 2025-11-04 09:03:17,466 INFO spawned: 'notificationworker' with pid 77 2025-11-04 09:03:17,468 INFO spawned: 'proxycacheblobworker' with pid 78 2025-11-04 09:03:17,471 INFO spawned: 'pullstatsredisflushworker' with pid 79 2025-11-04 09:03:17,479 INFO spawnerr: command at '/usr/local/bin/pushgateway' is a directory 2025-11-04 09:03:17,481 INFO spawned: 'queuecleanupworker' with pid 81 2025-11-04 09:03:17,483 INFO spawned: 'quotaregistrysizeworker' with pid 82 2025-11-04 09:03:17,485 INFO spawned: 'quotatotalworker' with pid 83 2025-11-04 09:03:17,559 INFO spawned: 'reconciliationworker' with pid 84 2025-11-04 09:03:17,572 INFO spawned: 'repositoryactioncounter' with pid 85 2025-11-04 09:03:17,581 INFO spawned: 'repositorygcworker' with pid 90 2025-11-04 09:03:17,660 INFO spawned: 'securityscanningnotificationworker' with pid 91 2025-11-04 09:03:17,667 INFO spawned: 'securityworker' with pid 92 2025-11-04 09:03:17,678 INFO spawned: 'servicekey' with pid 95 2025-11-04 09:03:17,680 INFO spawned: 'storagereplication' with pid 96 2025-11-04 09:03:17,761 INFO spawned: 'teamsyncworker' with pid 99 2025-11-04 09:03:18,377 INFO success: stdout entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: autopruneworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: blobuploadcleanupworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: builder entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: buildlogsarchiver entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: chunkcleanupworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: dnsmasq entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,377 INFO success: expiredappspecifictokenworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: exportactionlogsworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: gcworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: globalpromstats entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: gunicorn-registry entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: gunicorn-secscan entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,378 INFO success: gunicorn-web entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) dnsmasq stderr | dnsmasq: started, version 2.85 cachesize 150 dnsmasq stderr | dnsmasq: compile time options: IPv6 GNU-getopt DBus no-UBus no-i18n IDN2 DHCP DHCPv6 no-Lua TFTP no-conntrack ipset auth cryptohash DNSSEC loop-detect inotify dumpfile dnsmasq stderr | dnsmasq: reading /etc/resolv.conf dnsmasq stderr | dnsmasq: using nameserver 172.30.0.10#53 dnsmasq stderr | dnsmasq: read /etc/hosts - 7 addresses nginx stdout | 2025/11/04 09:03:17 [alert] 102#102: setpriority(-10) failed (13: Permission denied) nginx stdout | 2025/11/04 09:03:17 [alert] 103#103: setpriority(-10) failed (13: Permission denied) 2025-11-04 09:03:18,379 INFO success: logrotateworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) nginx stdout | 2025/11/04 09:03:17 [alert] 104#104: setpriority(-10) failed (13: Permission denied) nginx stdout | 2025/11/04 09:03:17 [alert] 101#101: setpriority(-10) failed (13: Permission denied) nginx stdout | 2025/11/04 09:03:17 [alert] 100#100: setpriority(-10) failed (13: Permission denied) nginx stdout | 2025/11/04 09:03:17 [alert] 106#106: setpriority(-10) failed (13: Permission denied) 2025-11-04 09:03:18,382 INFO success: manifestbackfillworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) nginx stdout | 2025/11/04 09:03:17 [alert] 105#105: setpriority(-10) failed (13: Permission denied) 2025-11-04 09:03:18,458 INFO success: manifestsubjectbackfillworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,458 INFO success: memcache entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) nginx stdout | 2025/11/04 09:03:17 [alert] 107#107: setpriority(-10) failed (13: Permission denied) manifestsubjectbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' manifestsubjectbackfillworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) 2025-11-04 09:03:18,667 INFO success: namespacegcworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: nginx entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: notificationworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: proxycacheblobworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: pullstatsredisflushworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO spawnerr: command at '/usr/local/bin/pushgateway' is a directory 2025-11-04 09:03:18,667 INFO success: queuecleanupworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: quotaregistrysizeworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: quotatotalworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: reconciliationworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: repositoryactioncounter entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: repositorygcworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: securityscanningnotificationworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:18,667 INFO success: securityworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) manifestsubjectbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' manifestsubjectbackfillworker stderr | return re.sub('[^\w]+', '_', self.name) 2025-11-04 09:03:19,064 INFO success: servicekey entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:19,064 INFO success: storagereplication entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) 2025-11-04 09:03:19,064 INFO success: teamsyncworker entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) manifestbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' manifestbackfillworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) manifestbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' manifestbackfillworker stderr | return re.sub('[^\w]+', '_', self.name) quotatotalworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' quotatotalworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) quotatotalworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' quotatotalworker stderr | return re.sub('[^\w]+', '_', self.name) 2025-11-04 09:03:21,059 INFO spawnerr: command at '/usr/local/bin/pushgateway' is a directory 2025-11-04 09:03:24,062 INFO spawnerr: command at '/usr/local/bin/pushgateway' is a directory 2025-11-04 09:03:24,062 INFO gave up: pushgateway entered FATAL state, too many start retries too quickly nginx stdout | 2025/11/04 09:03:25 [crit] 104#104: *1 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:03:25 [error] 104#104: *1 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:03:25 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) reconciliationworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay manifestsubjectbackfillworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay expiredappspecifictokenworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay reconciliationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' reconciliationworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) reconciliationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' reconciliationworker stderr | return re.sub('[^\w]+', '_', self.name) manifestsubjectbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' manifestsubjectbackfillworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') expiredappspecifictokenworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' expiredappspecifictokenworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) expiredappspecifictokenworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' expiredappspecifictokenworker stderr | return re.sub('[^\w]+', '_', self.name) reconciliationworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' reconciliationworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') expiredappspecifictokenworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' expiredappspecifictokenworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') builder stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay builder stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' builder stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) builder stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' builder stderr | return re.sub('[^\w]+', '_', self.name) chunkcleanupworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay builder stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' builder stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') buildlogsarchiver stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay teamsyncworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay globalpromstats stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay autopruneworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay storagereplication stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay blobuploadcleanupworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay queuecleanupworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay nginx stdout | 2025/11/04 09:03:40 [crit] 103#103: *3 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:03:40 [error] 103#103: *3 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:03:40 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) gcworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay notificationworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay repositorygcworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay exportactionlogsworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay quotaregistrysizeworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay securityscanningnotificationworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay securityworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay namespacegcworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay repositoryactioncounter stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay logrotateworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay gunicorn-secscan stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay manifestbackfillworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay proxycacheblobworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay servicekey stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay gunicorn-web stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay quotatotalworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay buildlogsarchiver stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' buildlogsarchiver stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) buildlogsarchiver stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' buildlogsarchiver stderr | return re.sub('[^\w]+', '_', self.name) chunkcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' chunkcleanupworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) chunkcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' chunkcleanupworker stderr | return re.sub('[^\w]+', '_', self.name) pullstatsredisflushworker stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay notificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' notificationworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) teamsyncworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' teamsyncworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) manifestsubjectbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' manifestsubjectbackfillworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) globalpromstats stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' globalpromstats stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) notificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' notificationworker stderr | return re.sub('[^\w]+', '_', self.name) blobuploadcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' blobuploadcleanupworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) repositoryactioncounter stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' repositoryactioncounter stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) autopruneworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' autopruneworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) buildlogsarchiver stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' buildlogsarchiver stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') globalpromstats stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' globalpromstats stderr | return re.sub('[^\w]+', '_', self.name) teamsyncworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' teamsyncworker stderr | return re.sub('[^\w]+', '_', self.name) queuecleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' queuecleanupworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) reconciliationworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' reconciliationworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) repositoryactioncounter stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' repositoryactioncounter stderr | return re.sub('[^\w]+', '_', self.name) autopruneworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' autopruneworker stderr | return re.sub('[^\w]+', '_', self.name) blobuploadcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' blobuploadcleanupworker stderr | return re.sub('[^\w]+', '_', self.name) queuecleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' queuecleanupworker stderr | return re.sub('[^\w]+', '_', self.name) pullstatsredisflushworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' pullstatsredisflushworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) securityscanningnotificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' securityscanningnotificationworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) repositorygcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' repositorygcworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) expiredappspecifictokenworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' expiredappspecifictokenworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) quotaregistrysizeworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' quotaregistrysizeworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) securityworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' securityworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) storagereplication stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' storagereplication stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) pullstatsredisflushworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' pullstatsredisflushworker stderr | return re.sub('[^\w]+', '_', self.name) repositorygcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' repositorygcworker stderr | return re.sub('[^\w]+', '_', self.name) securityscanningnotificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' securityscanningnotificationworker stderr | return re.sub('[^\w]+', '_', self.name) quotaregistrysizeworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' quotaregistrysizeworker stderr | return re.sub('[^\w]+', '_', self.name) notificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' notificationworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') securityworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' securityworker stderr | return re.sub('[^\w]+', '_', self.name) chunkcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' chunkcleanupworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') storagereplication stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' storagereplication stderr | return re.sub('[^\w]+', '_', self.name) gcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' gcworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) repositoryactioncounter stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' repositoryactioncounter stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') globalpromstats stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' globalpromstats stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') builder stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' builder stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) teamsyncworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' teamsyncworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') gcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' gcworker stderr | return re.sub('[^\w]+', '_', self.name) exportactionlogsworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' exportactionlogsworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) queuecleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' queuecleanupworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') autopruneworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' autopruneworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') blobuploadcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' blobuploadcleanupworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') namespacegcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' namespacegcworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) securityscanningnotificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' securityscanningnotificationworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') securityworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' securityworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') exportactionlogsworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' exportactionlogsworker stderr | return re.sub('[^\w]+', '_', self.name) quotatotalworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' quotatotalworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') logrotateworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' logrotateworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) namespacegcworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' namespacegcworker stderr | return re.sub('[^\w]+', '_', self.name) quotaregistrysizeworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' quotaregistrysizeworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') logrotateworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' logrotateworker stderr | return re.sub('[^\w]+', '_', self.name) manifestbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' manifestbackfillworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') pullstatsredisflushworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' pullstatsredisflushworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') repositorygcworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' repositorygcworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') storagereplication stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' storagereplication stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') proxycacheblobworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' proxycacheblobworker stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) servicekey stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' servicekey stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) gcworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' gcworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') proxycacheblobworker stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' proxycacheblobworker stderr | return re.sub('[^\w]+', '_', self.name) servicekey stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' servicekey stderr | return re.sub('[^\w]+', '_', self.name) exportactionlogsworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' exportactionlogsworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') namespacegcworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' namespacegcworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') logrotateworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' logrotateworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') proxycacheblobworker stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' proxycacheblobworker stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') servicekey stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' servicekey stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') nginx stdout | 2025/11/04 09:03:55 [crit] 102#102: *5 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:03:55 [error] 102#102: *5 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:03:55 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) builder stdout | 2025-11-04 09:04:01,061 [60] [DEBUG] [__main__] Building is disabled. Please enable the feature flag gunicorn-secscan stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' gunicorn-secscan stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,078 [73] [DEBUG] [workers.worker] Scheduling worker. manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,080 [73] [INFO] [apscheduler.scheduler] Scheduler started manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,083 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,085 [73] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,085 [73] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:00.086435 manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,158 [73] [INFO] [apscheduler.scheduler] Added job "ManifestSubjectBackfillWorker._backfill_manifest_subject" to job store "default" manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,158 [73] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:52.159714 manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,159 [73] [INFO] [apscheduler.scheduler] Added job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type" to job store "default" manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:04:03,159 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:52.159714+00:00 (in 49.000467 seconds) gunicorn-secscan stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' gunicorn-secscan stderr | return re.sub('[^\w]+', '_', self.name) expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,161 [64] [DEBUG] [__main__] Starting expired app specific token GC worker expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,162 [64] [DEBUG] [__main__] Found expiration window: 1d expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,165 [64] [DEBUG] [workers.worker] Scheduling worker. expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,167 [64] [INFO] [apscheduler.scheduler] Scheduler started expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,168 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,168 [64] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:10:01.169469 expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,170 [64] [INFO] [apscheduler.scheduler] Added job "ExpiredAppSpecificTokenWorker._gc_expired_tokens" to job store "default" expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,169 [64] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,171 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run expiredappspecifictokenworker stdout | 2025-11-04 09:04:04,172 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:01.169469+00:00 (in 356.996504 seconds) gunicorn-secscan stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' gunicorn-secscan stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') storagereplication stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' storagereplication stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) gunicorn-registry stderr | fatal: not a git repository: /quay-registry/../.git/modules/quay gunicorn-web stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' gunicorn-web stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) buildlogsarchiver stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' buildlogsarchiver stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) gunicorn-web stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' gunicorn-web stderr | return re.sub('[^\w]+', '_', self.name) nginx stdout | 2025/11/04 09:04:10 [crit] 100#100: *7 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:04:10 [error] 100#100: *7 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:04:10 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) notificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' notificationworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) repositoryactioncounter stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' repositoryactioncounter stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) gunicorn-web stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' gunicorn-web stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') quotatotalworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' quotatotalworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) queuecleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' queuecleanupworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) securityscanningnotificationworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' securityscanningnotificationworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) repositorygcworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' repositorygcworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) chunkcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' chunkcleanupworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) securityworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' securityworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) globalpromstats stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' globalpromstats stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) teamsyncworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' teamsyncworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) blobuploadcleanupworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' blobuploadcleanupworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) autopruneworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' autopruneworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) quotaregistrysizeworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' quotaregistrysizeworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) servicekey stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' servicekey stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) pullstatsredisflushworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' pullstatsredisflushworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) proxycacheblobworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' proxycacheblobworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) manifestbackfillworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' manifestbackfillworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) exportactionlogsworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' exportactionlogsworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) gcworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' gcworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) namespacegcworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' namespacegcworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) logrotateworker stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' logrotateworker stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) storagereplication stdout | 2025-11-04 09:04:24,478 [96] [DEBUG] [__main__] Full storage replication disabled; skipping nginx stdout | 2025/11/04 09:04:25 [crit] 101#101: *9 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:04:25 [error] 101#101: *9 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:04:25 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) gunicorn-secscan stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' gunicorn-secscan stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) notificationworker stdout | 2025-11-04 09:04:29,658 [77] [DEBUG] [workers.worker] Scheduling worker. notificationworker stdout | 2025-11-04 09:04:29,661 [77] [INFO] [apscheduler.scheduler] Scheduler started notificationworker stdout | 2025-11-04 09:04:29,764 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:29,764 [77] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added notificationworker stdout | 2025-11-04 09:04:29,765 [77] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:37.766253 notificationworker stdout | 2025-11-04 09:04:29,766 [77] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" notificationworker stdout | 2025-11-04 09:04:29,766 [77] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:07:19.767846 notificationworker stdout | 2025-11-04 09:04:29,767 [77] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" notificationworker stdout | 2025-11-04 09:04:29,767 [77] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:27.768179 notificationworker stdout | 2025-11-04 09:04:29,767 [77] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" notificationworker stdout | 2025-11-04 09:04:29,771 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:29,771 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:37.766253+00:00 (in 7.994333 seconds) builder stdout | 2025-11-04 09:04:30,067 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:04:30,966 [61] [DEBUG] [workers.worker] Scheduling worker. buildlogsarchiver stdout | 2025-11-04 09:04:30,966 [61] [INFO] [apscheduler.scheduler] Scheduler started buildlogsarchiver stdout | 2025-11-04 09:04:30,976 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:04:30,977 [61] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added buildlogsarchiver stdout | 2025-11-04 09:04:30,979 [61] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:44.980431 buildlogsarchiver stdout | 2025-11-04 09:04:30,981 [61] [INFO] [apscheduler.scheduler] Added job "ArchiveBuildLogsWorker._archive_redis_buildlogs" to job store "default" buildlogsarchiver stdout | 2025-11-04 09:04:30,982 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:04:30,983 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:44.980431+00:00 (in 13.997324 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:30,975 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:04:31,567 [67] [DEBUG] [workers.worker] Scheduling worker. globalpromstats stdout | 2025-11-04 09:04:31,568 [67] [INFO] [apscheduler.scheduler] Scheduler started globalpromstats stdout | 2025-11-04 09:04:31,579 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run globalpromstats stdout | 2025-11-04 09:04:31,579 [67] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:14:07.580640 globalpromstats stdout | 2025-11-04 09:04:31,581 [67] [INFO] [apscheduler.scheduler] Added job "GlobalPrometheusStatsWorker._try_report_stats" to job store "default" globalpromstats stdout | 2025-11-04 09:04:31,581 [67] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added globalpromstats stdout | 2025-11-04 09:04:31,659 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run globalpromstats stdout | 2025-11-04 09:04:31,659 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.580640+00:00 (in 575.920678 seconds) repositorygcworker stdout | 2025-11-04 09:04:32,074 [90] [DEBUG] [__main__] Starting repository GC worker repositorygcworker stdout | 2025-11-04 09:04:32,077 [90] [DEBUG] [workers.worker] Scheduling worker. repositorygcworker stdout | 2025-11-04 09:04:32,077 [90] [INFO] [apscheduler.scheduler] Scheduler started gunicorn-web stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' gunicorn-web stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) repositorygcworker stdout | 2025-11-04 09:04:32,179 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:32,179 [90] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:40.180734 repositorygcworker stdout | 2025-11-04 09:04:32,181 [90] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" repositorygcworker stdout | 2025-11-04 09:04:32,180 [90] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added repositorygcworker stdout | 2025-11-04 09:04:32,258 [90] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:41.259858 repositorygcworker stdout | 2025-11-04 09:04:32,264 [90] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" repositorygcworker stdout | 2025-11-04 09:04:32,264 [90] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:06.265965 repositorygcworker stdout | 2025-11-04 09:04:32,265 [90] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" repositorygcworker stdout | 2025-11-04 09:04:32,264 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:32,269 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:40.180734+00:00 (in 7.911011 seconds) repositorygcworker stdout | 2025-11-04 09:04:32,269 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:32,269 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:40.180734+00:00 (in 7.910865 seconds) expiredappspecifictokenworker stdout | 2025-11-04 09:04:32,278 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: gunicorn-registry stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:2784: SyntaxWarning: invalid escape sequence '\w' gunicorn-registry stderr | clean_field_names = re.sub('[^\w]+', '', '_'.join(accum)) gunicorn-registry stderr | /opt/app-root/lib64/python3.12/site-packages/peewee.py:5763: SyntaxWarning: invalid escape sequence '\w' gunicorn-registry stderr | return re.sub('[^\w]+', '_', self.name) gunicorn-registry stderr | /opt/app-root/lib64/python3.12/site-packages/playhouse/sqlite_ext.py:451: SyntaxWarning: invalid escape sequence '\s' gunicorn-registry stderr | _quote_re = re.compile('(?:[^\s"]|"(?:\\.|[^"])*")+') repositoryactioncounter stdout | 2025-11-04 09:04:35,359 [85] [DEBUG] [workers.worker] Scheduling worker. repositoryactioncounter stdout | 2025-11-04 09:04:35,359 [85] [INFO] [apscheduler.scheduler] Scheduler started repositoryactioncounter stdout | 2025-11-04 09:04:35,375 [85] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositoryactioncounter stdout | 2025-11-04 09:04:35,375 [85] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added repositoryactioncounter stdout | 2025-11-04 09:04:35,376 [85] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 11:27:48.377023 repositoryactioncounter stdout | 2025-11-04 09:04:35,377 [85] [INFO] [apscheduler.scheduler] Added job "RepositoryActionCountWorker._run_counting" to job store "default" repositoryactioncounter stdout | 2025-11-04 09:04:35,377 [85] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositoryactioncounter stdout | 2025-11-04 09:04:35,377 [85] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 11:27:48.377023+00:00 (in 8592.999696 seconds) chunkcleanupworker stdout | 2025-11-04 09:04:35,666 [62] [DEBUG] [__main__] Swift storage not detected; sleeping notificationworker stdout | 2025-11-04 09:04:37,767 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:37,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:37 GMT)" (scheduled at 2025-11-04 09:04:37.766253+00:00) notificationworker stdout | 2025-11-04 09:04:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:37,769 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 37, 768129), True, datetime.datetime(2025, 11, 4, 9, 4, 37, 768129), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:37,769 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:47.766253+00:00 (in 9.996588 seconds) notificationworker stdout | 2025-11-04 09:04:37,866 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:37,866 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:37,868 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:47 GMT)" executed successfully gunicorn-registry stderr | /opt/app-root/lib64/python3.12/site-packages/splunklib/client.py:782: SyntaxWarning: invalid escape sequence '\/' gunicorn-registry stderr | versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) teamsyncworker stdout | 2025-11-04 09:04:39,264 [99] [DEBUG] [__main__] Team syncing is disabled; sleeping queuecleanupworker stdout | 2025-11-04 09:04:39,275 [81] [DEBUG] [workers.worker] Scheduling worker. queuecleanupworker stdout | 2025-11-04 09:04:39,363 [81] [INFO] [apscheduler.scheduler] Scheduler started queuecleanupworker stdout | 2025-11-04 09:04:39,376 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run queuecleanupworker stdout | 2025-11-04 09:04:39,376 [81] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added queuecleanupworker stdout | 2025-11-04 09:04:39,377 [81] [DEBUG] [workers.worker] First run scheduled for 2025-11-05 01:11:34.378694 queuecleanupworker stdout | 2025-11-04 09:04:39,379 [81] [INFO] [apscheduler.scheduler] Added job "QueueCleanupWorker._cleanup_queue" to job store "default" queuecleanupworker stdout | 2025-11-04 09:04:39,379 [81] [DEBUG] [apscheduler.scheduler] Looking for jobs to run queuecleanupworker stdout | 2025-11-04 09:04:39,379 [81] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-05 01:11:34.378694+00:00 (in 58014.999257 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:04:39,476 [91] [DEBUG] [__main__] Starting security scanning notification worker securityscanningnotificationworker stdout | 2025-11-04 09:04:39,480 [91] [DEBUG] [workers.worker] Scheduling worker. securityscanningnotificationworker stdout | 2025-11-04 09:04:39,558 [91] [INFO] [apscheduler.scheduler] Scheduler started securityscanningnotificationworker stdout | 2025-11-04 09:04:39,565 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:04:39,565 [91] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added securityscanningnotificationworker stdout | 2025-11-04 09:04:39,567 [91] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:33.568037 securityscanningnotificationworker stdout | 2025-11-04 09:04:39,568 [91] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" securityscanningnotificationworker stdout | 2025-11-04 09:04:39,568 [91] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:07:28.569177 securityscanningnotificationworker stdout | 2025-11-04 09:04:39,568 [91] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" securityscanningnotificationworker stdout | 2025-11-04 09:04:39,568 [91] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:06.569529 securityscanningnotificationworker stdout | 2025-11-04 09:04:39,568 [91] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" securityscanningnotificationworker stdout | 2025-11-04 09:04:39,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:04:39,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:06.569529+00:00 (in 26.998893 seconds) nginx stdout | 2025/11/04 09:04:40 [crit] 106#106: *11 connect() to unix:/tmp/gunicorn_web.sock failed (2: No such file or directory) while connecting to upstream, client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock:/health/instance", host: "10.131.2.29:8080" nginx stdout | 2025/11/04 09:04:40 [error] 106#106: *11 open() "/quay-registry/static/patternfly/quay-registry/static/502.html" failed (2: No such file or directory), client: 10.131.2.2, server: , request: "GET /health/instance HTTP/1.1", upstream: "http://unix:/tmp/gunicorn_web.sock/health/instance", host: "10.131.2.29:8080" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:04:40 +0000] "GET /health/instance HTTP/1.1" 404 146 "-" "kube-probe/1.32" (0.000 118 0.000) pullstatsredisflushworker stdout | 2025-11-04 09:04:40,074 [79] [INFO] [__main__] RedisFlushWorker: Initialized Redis client for pull metrics pullstatsredisflushworker stdout | 2025-11-04 09:04:40,076 [79] [DEBUG] [workers.worker] Scheduling worker. pullstatsredisflushworker stdout | 2025-11-04 09:04:40,076 [79] [INFO] [apscheduler.scheduler] Scheduler started pullstatsredisflushworker stdout | 2025-11-04 09:04:40,080 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:04:40,080 [79] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:02.081579 pullstatsredisflushworker stdout | 2025-11-04 09:04:40,082 [79] [INFO] [apscheduler.scheduler] Added job "RedisFlushWorker._flush_pull_metrics" to job store "default" pullstatsredisflushworker stdout | 2025-11-04 09:04:40,081 [79] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added pullstatsredisflushworker stdout | 2025-11-04 09:04:40,084 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:04:40,084 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:02.081579+00:00 (in 21.996852 seconds) repositorygcworker stdout | 2025-11-04 09:04:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:40,271 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:40 GMT)" (scheduled at 2025-11-04 09:04:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:04:40,271 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:41.259858+00:00 (in 0.987971 seconds) repositorygcworker stdout | 2025-11-04 09:04:40,273 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:04:40,275 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 40, 273725), True, datetime.datetime(2025, 11, 4, 9, 4, 40, 273725), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:04:40,359 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:04:40,359 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:04:40,359 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:40 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:04:41,261 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:04:41,262 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:06.265965+00:00 (in 25.003980 seconds) repositorygcworker stdout | 2025-11-04 09:04:41,262 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:41 GMT)" (scheduled at 2025-11-04 09:04:41.259858+00:00) repositorygcworker stdout | 2025-11-04 09:04:41,263 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:41,273 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), True, datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:41,275 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), True, datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 4, 41, 262381), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:04:41,276 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:04:41,279 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:09:41 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:04:41,677 [69] [DEBUG] [sentry_sdk.errors] [Tracing] Create new propagation context: {'trace_id': '5b76451f2ab542e4b9f69ec0c97abca7', 'span_id': 'bc8ae1c4ed7fd79d', 'parent_span_id': None, 'dynamic_sampling_context': None} gunicorn-secscan stdout | 2025-11-04 09:04:41,768 [69] [DEBUG] [app] Loading default config. gunicorn-secscan stdout | 2025-11-04 09:04:41,770 [69] [DEBUG] [util.config.provider.basefileprovider] Applying config file: /quay-registry/conf/stack/config.yaml gunicorn-secscan stdout | 2025-11-04 09:04:41,784 [69] [DEBUG] [app] Loaded config gunicorn-secscan stdout | 2025-11-04 09:04:41,861 [69] [INFO] [util.ipresolver] Loading AWS IP ranges from disk gunicorn-secscan stdout | 2025-11-04 09:04:41,880 [69] [DEBUG] [util.ipresolver] Building AWS IP ranges manifestbackfillworker stdout | 2025-11-04 09:04:41,980 [72] [DEBUG] [workers.worker] Scheduling worker. manifestbackfillworker stdout | 2025-11-04 09:04:42,058 [72] [INFO] [apscheduler.scheduler] Scheduler started manifestbackfillworker stdout | 2025-11-04 09:04:42,071 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestbackfillworker stdout | 2025-11-04 09:04:42,071 [72] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added manifestbackfillworker stdout | 2025-11-04 09:04:42,072 [72] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:39:53.073354 manifestbackfillworker stdout | 2025-11-04 09:04:42,073 [72] [INFO] [apscheduler.scheduler] Added job "ManifestBackfillWorker._backfill_manifests" to job store "default" manifestbackfillworker stdout | 2025-11-04 09:04:42,073 [72] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestbackfillworker stdout | 2025-11-04 09:04:42,073 [72] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:39:53.073354+00:00 (in 2110.999804 seconds) gunicorn-secscan stdout | 2025-11-04 09:04:42,370 [69] [DEBUG] [util.ipresolver] Finished building AWS IP ranges gunicorn-secscan stdout | 2025-11-04 09:04:42,374 [69] [DEBUG] [botocore.hooks] Changing event name from creating-client-class.iot-data to creating-client-class.iot-data-plane gunicorn-secscan stdout | 2025-11-04 09:04:42,378 [69] [DEBUG] [botocore.hooks] Changing event name from before-call.apigateway to before-call.api-gateway gunicorn-secscan stdout | 2025-11-04 09:04:42,380 [69] [DEBUG] [botocore.hooks] Changing event name from request-created.machinelearning.Predict to request-created.machine-learning.Predict gunicorn-secscan stdout | 2025-11-04 09:04:42,384 [69] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.autoscaling.CreateLaunchConfiguration to before-parameter-build.auto-scaling.CreateLaunchConfiguration gunicorn-secscan stdout | 2025-11-04 09:04:42,458 [69] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.route53 to before-parameter-build.route-53 gunicorn-secscan stdout | 2025-11-04 09:04:42,459 [69] [DEBUG] [botocore.hooks] Changing event name from request-created.cloudsearchdomain.Search to request-created.cloudsearch-domain.Search gunicorn-secscan stdout | 2025-11-04 09:04:42,461 [69] [DEBUG] [botocore.hooks] Changing event name from docs.*.autoscaling.CreateLaunchConfiguration.complete-section to docs.*.auto-scaling.CreateLaunchConfiguration.complete-section gunicorn-secscan stdout | 2025-11-04 09:04:42,466 [69] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.logs.CreateExportTask to before-parameter-build.cloudwatch-logs.CreateExportTask gunicorn-secscan stdout | 2025-11-04 09:04:42,467 [69] [DEBUG] [botocore.hooks] Changing event name from docs.*.logs.CreateExportTask.complete-section to docs.*.cloudwatch-logs.CreateExportTask.complete-section gunicorn-secscan stdout | 2025-11-04 09:04:42,468 [69] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.cloudsearchdomain.Search to before-parameter-build.cloudsearch-domain.Search gunicorn-secscan stdout | 2025-11-04 09:04:42,469 [69] [DEBUG] [botocore.hooks] Changing event name from docs.*.cloudsearchdomain.Search.complete-section to docs.*.cloudsearch-domain.Search.complete-section logrotateworker stdout | 2025-11-04 09:04:42,584 [71] [DEBUG] [__main__] Action log rotation worker not enabled; skipping gunicorn-secscan stdout | 2025-11-04 09:04:42,678 [69] [DEBUG] [data.database] Configuring database gunicorn-secscan stdout | 2025-11-04 09:04:42,680 [69] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-secscan gunicorn-secscan stdout | 2025-11-04 09:04:42,681 [69] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [INFO] [data.secscan_model] =============================== gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [INFO] [data.secscan_model] Using split secscan model: `[]` gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [INFO] [data.secscan_model] =============================== gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [DEBUG] [data.logs_model] Configuring log model gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [INFO] [data.logs_model] =============================== gunicorn-secscan stdout | 2025-11-04 09:04:42,683 [69] [INFO] [data.logs_model] Using logs model `` gunicorn-secscan stdout | 2025-11-04 09:04:42,684 [69] [INFO] [data.logs_model] =============================== gunicorn-web stdout | 2025-11-04 09:04:44,180 [70] [DEBUG] [sentry_sdk.errors] [Tracing] Create new propagation context: {'trace_id': '622ce00c0cc94dde90a2f383181cdc16', 'span_id': 'a2ac9a6a682c7496', 'parent_span_id': None, 'dynamic_sampling_context': None} gunicorn-web stdout | 2025-11-04 09:04:44,275 [70] [DEBUG] [app] Loading default config. gunicorn-web stdout | 2025-11-04 09:04:44,277 [70] [DEBUG] [util.config.provider.basefileprovider] Applying config file: /quay-registry/conf/stack/config.yaml gunicorn-web stdout | 2025-11-04 09:04:44,359 [70] [DEBUG] [app] Loaded config gunicorn-web stdout | 2025-11-04 09:04:44,362 [70] [INFO] [util.ipresolver] Loading AWS IP ranges from disk gunicorn-web stdout | 2025-11-04 09:04:44,380 [70] [DEBUG] [util.ipresolver] Building AWS IP ranges gunicorn-secscan stdout | 2025-11-04 09:04:44,465 [69] [DEBUG] [__config__] Starting secscan gunicorn with 2 workers and gevent worker class gunicorn-secscan stderr | Traceback (most recent call last): gunicorn-secscan stderr | File "src/gevent/_abstract_linkable.py", line 287, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-secscan stderr | File "src/gevent/_abstract_linkable.py", line 333, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-secscan stderr | AssertionError: (None, ) gunicorn-secscan stderr | 2025-11-04T09:04:44Z gunicorn-secscan stderr | gunicorn-secscan stderr | failed with AssertionError proxycacheblobworker stdout | 2025-11-04 09:04:44,668 [78] [DEBUG] [__main__] Starting proxy cache blob worker proxycacheblobworker stdout | 2025-11-04 09:04:44,672 [78] [DEBUG] [workers.worker] Scheduling worker. proxycacheblobworker stdout | 2025-11-04 09:04:44,673 [78] [INFO] [apscheduler.scheduler] Scheduler started proxycacheblobworker stdout | 2025-11-04 09:04:44,675 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:44,675 [78] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:49.676299 proxycacheblobworker stdout | 2025-11-04 09:04:44,678 [78] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" proxycacheblobworker stdout | 2025-11-04 09:04:44,678 [78] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:09:44.679698 proxycacheblobworker stdout | 2025-11-04 09:04:44,678 [78] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" proxycacheblobworker stdout | 2025-11-04 09:04:44,678 [78] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:47.679968 proxycacheblobworker stdout | 2025-11-04 09:04:44,679 [78] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" proxycacheblobworker stdout | 2025-11-04 09:04:44,677 [78] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added proxycacheblobworker stdout | 2025-11-04 09:04:44,681 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:44,682 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:47.679968+00:00 (in 2.997596 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:04:44,766 [82] [DEBUG] [workers.worker] Scheduling worker. quotaregistrysizeworker stdout | 2025-11-04 09:04:44,768 [82] [INFO] [apscheduler.scheduler] Scheduler started quotaregistrysizeworker stdout | 2025-11-04 09:04:44,769 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:04:44,769 [82] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:07.770322 quotaregistrysizeworker stdout | 2025-11-04 09:04:44,771 [82] [INFO] [apscheduler.scheduler] Added job "QuotaRegistrySizeWorker._calculate_registry_size" to job store "default" quotaregistrysizeworker stdout | 2025-11-04 09:04:44,770 [82] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added quotaregistrysizeworker stdout | 2025-11-04 09:04:44,773 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:04:44,774 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:07.770322+00:00 (in 22.996012 seconds) blobuploadcleanupworker stdout | 2025-11-04 09:04:44,779 [59] [DEBUG] [workers.worker] Scheduling worker. blobuploadcleanupworker stdout | 2025-11-04 09:04:44,780 [59] [INFO] [apscheduler.scheduler] Scheduler started blobuploadcleanupworker stdout | 2025-11-04 09:04:44,871 [59] [DEBUG] [apscheduler.scheduler] Looking for jobs to run blobuploadcleanupworker stdout | 2025-11-04 09:04:44,871 [59] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 10:04:10.872668 blobuploadcleanupworker stdout | 2025-11-04 09:04:44,872 [59] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added blobuploadcleanupworker stdout | 2025-11-04 09:04:44,873 [59] [INFO] [apscheduler.scheduler] Added job "BlobUploadCleanupWorker._try_cleanup_uploads" to job store "default" blobuploadcleanupworker stdout | 2025-11-04 09:04:44,873 [59] [DEBUG] [apscheduler.scheduler] Looking for jobs to run blobuploadcleanupworker stdout | 2025-11-04 09:04:44,873 [59] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:04:10.872668+00:00 (in 3565.999078 seconds) gunicorn-web stdout | 2025-11-04 09:04:44,880 [70] [DEBUG] [util.ipresolver] Finished building AWS IP ranges gunicorn-web stdout | 2025-11-04 09:04:44,958 [70] [DEBUG] [botocore.hooks] Changing event name from creating-client-class.iot-data to creating-client-class.iot-data-plane gunicorn-web stdout | 2025-11-04 09:04:44,962 [70] [DEBUG] [botocore.hooks] Changing event name from before-call.apigateway to before-call.api-gateway gunicorn-web stdout | 2025-11-04 09:04:44,964 [70] [DEBUG] [botocore.hooks] Changing event name from request-created.machinelearning.Predict to request-created.machine-learning.Predict gunicorn-web stdout | 2025-11-04 09:04:44,968 [70] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.autoscaling.CreateLaunchConfiguration to before-parameter-build.auto-scaling.CreateLaunchConfiguration gunicorn-web stdout | 2025-11-04 09:04:44,969 [70] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.route53 to before-parameter-build.route-53 gunicorn-web stdout | 2025-11-04 09:04:44,971 [70] [DEBUG] [botocore.hooks] Changing event name from request-created.cloudsearchdomain.Search to request-created.cloudsearch-domain.Search gunicorn-web stdout | 2025-11-04 09:04:44,973 [70] [DEBUG] [botocore.hooks] Changing event name from docs.*.autoscaling.CreateLaunchConfiguration.complete-section to docs.*.auto-scaling.CreateLaunchConfiguration.complete-section gunicorn-web stdout | 2025-11-04 09:04:44,978 [70] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.logs.CreateExportTask to before-parameter-build.cloudwatch-logs.CreateExportTask gunicorn-web stdout | 2025-11-04 09:04:44,979 [70] [DEBUG] [botocore.hooks] Changing event name from docs.*.logs.CreateExportTask.complete-section to docs.*.cloudwatch-logs.CreateExportTask.complete-section gunicorn-web stdout | 2025-11-04 09:04:44,980 [70] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.cloudsearchdomain.Search to before-parameter-build.cloudsearch-domain.Search buildlogsarchiver stdout | 2025-11-04 09:04:44,981 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gunicorn-web stdout | 2025-11-04 09:04:44,982 [70] [DEBUG] [botocore.hooks] Changing event name from docs.*.cloudsearchdomain.Search.complete-section to docs.*.cloudsearch-domain.Search.complete-section buildlogsarchiver stdout | 2025-11-04 09:04:45,068 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:44 GMT)" (scheduled at 2025-11-04 09:04:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:04:45,068 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:14.980431+00:00 (in 29.911538 seconds) buildlogsarchiver stdout | 2025-11-04 09:04:45,070 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 4, 45, 69358), False, 50, 1, 0]) autopruneworker stdout | 2025-11-04 09:04:45,075 [58] [DEBUG] [workers.worker] Scheduling worker. autopruneworker stdout | 2025-11-04 09:04:45,075 [58] [INFO] [apscheduler.scheduler] Scheduler started autopruneworker stdout | 2025-11-04 09:04:45,076 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:04:45,076 [58] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:07.077358 autopruneworker stdout | 2025-11-04 09:04:45,077 [58] [INFO] [apscheduler.scheduler] Added job "AutoPruneWorker.prune" to job store "default" autopruneworker stdout | 2025-11-04 09:04:45,076 [58] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added autopruneworker stdout | 2025-11-04 09:04:45,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:04:45,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:07.077358+00:00 (in 21.999734 seconds) buildlogsarchiver stdout | 2025-11-04 09:04:45,087 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:04:45,158 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:04:45,162 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:45,166 [70] [DEBUG] [data.database] Configuring database gunicorn-web stdout | 2025-11-04 09:04:45,167 [70] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:45,167 [70] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:45,168 [70] [INFO] [data.secscan_model] =============================== gunicorn-web stdout | 2025-11-04 09:04:45,168 [70] [INFO] [data.secscan_model] Using split secscan model: `[]` gunicorn-web stdout | 2025-11-04 09:04:45,168 [70] [INFO] [data.secscan_model] =============================== gunicorn-web stdout | 2025-11-04 09:04:45,168 [70] [DEBUG] [data.logs_model] Configuring log model gunicorn-web stdout | 2025-11-04 09:04:45,169 [70] [INFO] [data.logs_model] =============================== gunicorn-web stdout | 2025-11-04 09:04:45,169 [70] [INFO] [data.logs_model] Using logs model `` gunicorn-web stdout | 2025-11-04 09:04:45,169 [70] [INFO] [data.logs_model] =============================== servicekey stdout | 2025-11-04 09:04:45,363 [95] [DEBUG] [workers.worker] Scheduling worker. servicekey stdout | 2025-11-04 09:04:45,363 [95] [INFO] [apscheduler.scheduler] Scheduler started servicekey stdout | 2025-11-04 09:04:45,363 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run servicekey stdout | 2025-11-04 09:04:45,363 [95] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:27:59.364881 servicekey stdout | 2025-11-04 09:04:45,364 [95] [INFO] [apscheduler.scheduler] Added job "ServiceKeyWorker._refresh_service_key" to job store "default" servicekey stdout | 2025-11-04 09:04:45,364 [95] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added servicekey stdout | 2025-11-04 09:04:45,365 [95] [DEBUG] [apscheduler.scheduler] Looking for jobs to run servicekey stdout | 2025-11-04 09:04:45,365 [95] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:27:59.364881+00:00 (in 1393.999742 seconds) exportactionlogsworker stdout | 2025-11-04 09:04:45,476 [65] [DEBUG] [__main__] Starting export action logs worker exportactionlogsworker stdout | 2025-11-04 09:04:45,478 [65] [DEBUG] [workers.worker] Scheduling worker. exportactionlogsworker stdout | 2025-11-04 09:04:45,479 [65] [INFO] [apscheduler.scheduler] Scheduler started exportactionlogsworker stdout | 2025-11-04 09:04:45,479 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:04:45,479 [65] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added exportactionlogsworker stdout | 2025-11-04 09:04:45,479 [65] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:38.480592 exportactionlogsworker stdout | 2025-11-04 09:04:45,480 [65] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" exportactionlogsworker stdout | 2025-11-04 09:04:45,480 [65] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:55.481710 exportactionlogsworker stdout | 2025-11-04 09:04:45,481 [65] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" exportactionlogsworker stdout | 2025-11-04 09:04:45,481 [65] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:12.482065 exportactionlogsworker stdout | 2025-11-04 09:04:45,481 [65] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" exportactionlogsworker stdout | 2025-11-04 09:04:45,481 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:04:45,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:12.482065+00:00 (in 27.000101 seconds) namespacegcworker stdout | 2025-11-04 09:04:45,493 [75] [DEBUG] [__main__] Starting namespace GC worker namespacegcworker stdout | 2025-11-04 09:04:45,495 [75] [DEBUG] [workers.worker] Scheduling worker. namespacegcworker stdout | 2025-11-04 09:04:45,558 [75] [INFO] [apscheduler.scheduler] Scheduler started namespacegcworker stdout | 2025-11-04 09:04:45,558 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:04:45,559 [75] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added namespacegcworker stdout | 2025-11-04 09:04:45,559 [75] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:53.560131 namespacegcworker stdout | 2025-11-04 09:04:45,560 [75] [INFO] [apscheduler.scheduler] Added job "QueueWorker.poll_queue" to job store "default" namespacegcworker stdout | 2025-11-04 09:04:45,560 [75] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:07:32.561493 namespacegcworker stdout | 2025-11-04 09:04:45,560 [75] [INFO] [apscheduler.scheduler] Added job "QueueWorker.update_queue_metrics" to job store "default" namespacegcworker stdout | 2025-11-04 09:04:45,560 [75] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:33.561829 namespacegcworker stdout | 2025-11-04 09:04:45,561 [75] [INFO] [apscheduler.scheduler] Added job "QueueWorker.run_watchdog" to job store "default" namespacegcworker stdout | 2025-11-04 09:04:45,561 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:04:45,561 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:53.560131+00:00 (in 7.998827 seconds) gcworker stdout | 2025-11-04 09:04:45,659 [66] [DEBUG] [workers.worker] Scheduling worker. gcworker stdout | 2025-11-04 09:04:45,660 [66] [INFO] [apscheduler.scheduler] Scheduler started gcworker stdout | 2025-11-04 09:04:45,660 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:45,660 [66] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added gcworker stdout | 2025-11-04 09:04:45,660 [66] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:55.661677 gcworker stdout | 2025-11-04 09:04:45,662 [66] [INFO] [apscheduler.scheduler] Added job "GarbageCollectionWorker._garbage_collection_repos" to job store "default" gcworker stdout | 2025-11-04 09:04:45,662 [66] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:07.663502 gcworker stdout | 2025-11-04 09:04:45,662 [66] [INFO] [apscheduler.scheduler] Added job "GarbageCollectionWorker._scan_notifications" to job store "default" gcworker stdout | 2025-11-04 09:04:45,662 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:45,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:55.661677+00:00 (in 9.998657 seconds) securityworker stdout | 2025-11-04 09:04:45,704 [92] [DEBUG] [workers.worker] Scheduling worker. securityworker stdout | 2025-11-04 09:04:45,705 [92] [INFO] [apscheduler.scheduler] Scheduler started securityworker stdout | 2025-11-04 09:04:45,705 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:45,705 [92] [DEBUG] [apscheduler.scheduler] No jobs; waiting until a job is added securityworker stdout | 2025-11-04 09:04:45,705 [92] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:04:49.706478 securityworker stdout | 2025-11-04 09:04:45,706 [92] [INFO] [apscheduler.scheduler] Added job "SecurityWorker._index_in_scanner" to job store "default" securityworker stdout | 2025-11-04 09:04:45,706 [92] [DEBUG] [workers.worker] First run scheduled for 2025-11-04 09:05:01.707296 securityworker stdout | 2025-11-04 09:04:45,706 [92] [INFO] [apscheduler.scheduler] Added job "SecurityWorker._index_recent_manifests_in_scanner" to job store "default" securityworker stdout | 2025-11-04 09:04:45,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:45,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:49.706478+00:00 (in 3.999768 seconds) gunicorn-registry stdout | 2025-11-04 09:04:45,785 [68] [DEBUG] [sentry_sdk.errors] [Tracing] Create new propagation context: {'trace_id': '4c1845c7861447f2b3940c9bcd532e74', 'span_id': 'b1983586bf0a998b', 'parent_span_id': None, 'dynamic_sampling_context': None} gunicorn-registry stdout | 2025-11-04 09:04:45,798 [68] [DEBUG] [app] Loading default config. gunicorn-registry stdout | 2025-11-04 09:04:45,798 [68] [DEBUG] [util.config.provider.basefileprovider] Applying config file: /quay-registry/conf/stack/config.yaml gunicorn-registry stdout | 2025-11-04 09:04:45,805 [68] [DEBUG] [app] Loaded config gunicorn-registry stdout | 2025-11-04 09:04:45,807 [68] [INFO] [util.ipresolver] Loading AWS IP ranges from disk gunicorn-registry stdout | 2025-11-04 09:04:45,818 [68] [DEBUG] [util.ipresolver] Building AWS IP ranges gunicorn-registry stdout | 2025-11-04 09:04:45,880 [68] [DEBUG] [util.ipresolver] Finished building AWS IP ranges gunicorn-registry stdout | 2025-11-04 09:04:45,883 [68] [DEBUG] [botocore.hooks] Changing event name from creating-client-class.iot-data to creating-client-class.iot-data-plane gunicorn-registry stdout | 2025-11-04 09:04:45,885 [68] [DEBUG] [botocore.hooks] Changing event name from before-call.apigateway to before-call.api-gateway gunicorn-registry stdout | 2025-11-04 09:04:45,886 [68] [DEBUG] [botocore.hooks] Changing event name from request-created.machinelearning.Predict to request-created.machine-learning.Predict gunicorn-registry stdout | 2025-11-04 09:04:45,887 [68] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.autoscaling.CreateLaunchConfiguration to before-parameter-build.auto-scaling.CreateLaunchConfiguration gunicorn-registry stdout | 2025-11-04 09:04:45,887 [68] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.route53 to before-parameter-build.route-53 gunicorn-registry stdout | 2025-11-04 09:04:45,888 [68] [DEBUG] [botocore.hooks] Changing event name from request-created.cloudsearchdomain.Search to request-created.cloudsearch-domain.Search gunicorn-registry stdout | 2025-11-04 09:04:45,889 [68] [DEBUG] [botocore.hooks] Changing event name from docs.*.autoscaling.CreateLaunchConfiguration.complete-section to docs.*.auto-scaling.CreateLaunchConfiguration.complete-section gunicorn-registry stdout | 2025-11-04 09:04:45,892 [68] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.logs.CreateExportTask to before-parameter-build.cloudwatch-logs.CreateExportTask gunicorn-registry stdout | 2025-11-04 09:04:45,892 [68] [DEBUG] [botocore.hooks] Changing event name from docs.*.logs.CreateExportTask.complete-section to docs.*.cloudwatch-logs.CreateExportTask.complete-section gunicorn-registry stdout | 2025-11-04 09:04:45,892 [68] [DEBUG] [botocore.hooks] Changing event name from before-parameter-build.cloudsearchdomain.Search to before-parameter-build.cloudsearch-domain.Search gunicorn-registry stdout | 2025-11-04 09:04:45,892 [68] [DEBUG] [botocore.hooks] Changing event name from docs.*.cloudsearchdomain.Search.complete-section to docs.*.cloudsearch-domain.Search.complete-section gunicorn-registry stdout | 2025-11-04 09:04:45,962 [68] [DEBUG] [data.database] Configuring database gunicorn-registry stdout | 2025-11-04 09:04:45,963 [68] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-registry gunicorn-registry stdout | 2025-11-04 09:04:45,963 [68] [INFO] [data.database] Connection pooling enabled for postgresql; stale timeout: None; max connection count: None gunicorn-registry stdout | 2025-11-04 09:04:45,964 [68] [INFO] [data.secscan_model] =============================== gunicorn-registry stdout | 2025-11-04 09:04:45,964 [68] [INFO] [data.secscan_model] Using split secscan model: `[]` gunicorn-registry stdout | 2025-11-04 09:04:45,964 [68] [INFO] [data.secscan_model] =============================== gunicorn-registry stdout | 2025-11-04 09:04:45,964 [68] [DEBUG] [data.logs_model] Configuring log model gunicorn-registry stdout | 2025-11-04 09:04:45,964 [68] [INFO] [data.logs_model] =============================== gunicorn-registry stdout | 2025-11-04 09:04:45,965 [68] [INFO] [data.logs_model] Using logs model `` gunicorn-registry stdout | 2025-11-04 09:04:45,965 [68] [INFO] [data.logs_model] =============================== gunicorn-registry stdout | 2025-11-04 09:04:46,832 [68] [DEBUG] [__config__] Starting registry gunicorn with 8 workers and gevent worker class gunicorn-registry stderr | Traceback (most recent call last): gunicorn-registry stderr | File "src/gevent/_abstract_linkable.py", line 287, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-registry stderr | File "src/gevent/_abstract_linkable.py", line 333, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-registry stderr | AssertionError: (None, ) gunicorn-registry stderr | 2025-11-04T09:04:46Z failed with AssertionError gunicorn-web stdout | 2025-11-04 09:04:47,100 [70] [DEBUG] [__config__] Starting web gunicorn with 4 workers and gevent worker class gunicorn-web stderr | Traceback (most recent call last): gunicorn-web stderr | File "src/gevent/_abstract_linkable.py", line 287, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-web stderr | File "src/gevent/_abstract_linkable.py", line 333, in gevent._gevent_c_abstract_linkable.AbstractLinkable._notify_links gunicorn-web stderr | AssertionError: (None, ) gunicorn-web stderr | 2025-11-04T09:04:47Z failed with AssertionError proxycacheblobworker stdout | 2025-11-04 09:04:47,681 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:47,681 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:47 GMT)" (scheduled at 2025-11-04 09:04:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:04:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:47 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:04:47,682 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:49.676299+00:00 (in 1.993782 seconds) notificationworker stdout | 2025-11-04 09:04:47,770 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:47,770 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:57.766253+00:00 (in 9.995520 seconds) notificationworker stdout | 2025-11-04 09:04:47,770 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:57 GMT)" (scheduled at 2025-11-04 09:04:47.766253+00:00) notificationworker stdout | 2025-11-04 09:04:47,771 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:47,771 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 47, 771116), True, datetime.datetime(2025, 11, 4, 9, 4, 47, 771116), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:47,778 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:47,778 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:47,778 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:04:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:04:59.676299+00:00 (in 9.999461 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:59 GMT)" (scheduled at 2025-11-04 09:04:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 49, 677229), True, datetime.datetime(2025, 11, 4, 9, 4, 49, 677229), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:49,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:04:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:04:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:04:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:49 GMT)" (scheduled at 2025-11-04 09:04:49.706478+00:00) securityworker stdout | 2025-11-04 09:04:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:04:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:04:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:01.707296+00:00 (in 11.999529 seconds) securityworker stdout | 2025-11-04 09:04:49,709 [92] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 securityworker stdout | 2025-11-04 09:04:49,718 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:04:49,720 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:04:49,727 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:04:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:04:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 23, 33]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-49 securityworker stdout | 2025-11-04 09:04:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:04:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 37, 47]) securityworker stdout | 2025-11-04 09:04:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:04:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Right range 23-33 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-23 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-23 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:04:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 3, 13]) securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-37 securityworker stdout | 2025-11-04 09:04:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-37 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stdout | 2025-11-04 09:04:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 33, 43]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 23-33 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Already merged with block 23-33 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 23-33 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,736 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:04:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 23-47 securityworker stdout | 2025-11-04 09:04:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-23 securityworker stdout | 2025-11-04 09:04:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 23-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-23 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:04:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 13, 23]) securityworker stdout | 2025-11-04 09:04:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:04:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-47 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-47 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Left range 3-47 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:04:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 47, 49]) securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-47 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-47 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 3 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 3 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:04:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:04:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:04:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:04:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:04:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 34, 44]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-34 securityworker stdout | 2025-11-04 09:04:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-34 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 5, 15]) securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,745 [92] [DEBUG] [util.migrate.allocator] Left range 5-15 securityworker stdout | 2025-11-04 09:04:49,745 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 5-15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-34 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:04:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-34 securityworker stdout | 2025-11-04 09:04:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:04:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 16, 26]) securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-34 securityworker stdout | 2025-11-04 09:04:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-34 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:04:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 26, 36]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:04:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:49,749 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 1, 11]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Left range 16-44 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:04:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 44, 49]) securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-44 securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-44 securityworker stdout | 2025-11-04 09:04:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-44 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 15-16 securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-16 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-16 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-16 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:04:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 15, 25]) securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 25-15 securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 25-15 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:04:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 25, 35]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:04:49,755 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:04:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:04:49,755 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:04:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:04:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 7, 17]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stdout | 2025-11-04 09:04:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:04:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 1, 11]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:04:49,759 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 36, 46]) securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Right range 36-46 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Right range 36-46 securityworker stdout | 2025-11-04 09:04:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-36 securityworker stdout | 2025-11-04 09:04:49,762 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-36 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stdout | 2025-11-04 09:04:49,762 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 35, 45]) securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Merging with block 36-46 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-46 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 36-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Left range 25-46 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:04:49,763 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stdout | 2025-11-04 09:04:49,764 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 46, 49]) securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-46 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-46 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-46 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 25 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Total range: 17-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-46 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-25 securityworker stdout | 2025-11-04 09:04:49,765 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-25 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:04:49,766 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 8, 59, 49, 720588), 17, 27]) securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stderr | 2025-11-04 09:04:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:04:49,767 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:04:49,767 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:04:50,786 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:52 GMT)" (scheduled at 2025-11-04 09:04:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,161 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:00.086435+00:00 (in 7.924990 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:04:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:04:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:04:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:04:53 GMT)" (scheduled at 2025-11-04 09:04:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:04:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:04:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 53, 561010), True, datetime.datetime(2025, 11, 4, 9, 4, 53, 561010), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:04:53,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:33.561829+00:00 (in 39.999350 seconds) namespacegcworker stdout | 2025-11-04 09:04:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:04:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:04:53,569 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:04:55,069 [257] [DEBUG] [app] Starting request: urn:request:290ea7f2-96e0-4a01-afce-9c870d815251 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:04:55,077 [257] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): localhost:8080 gunicorn-registry stdout | 2025-11-04 09:04:55,089 [250] [DEBUG] [app] Starting request: urn:request:2f296c05-c35c-4b7c-b739-d529026204d2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:04:55,090 [250] [DEBUG] [app] Ending request: urn:request:2f296c05-c35c-4b7c-b739-d529026204d2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2f296c05-c35c-4b7c-b739-d529026204d2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.006 162 0.006) gunicorn-registry stdout | 2025-11-04 09:04:55,092 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:04:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:55,092 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:55,093 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:55,099 [254] [DEBUG] [app] Starting request: urn:request:95b33c02-1ffb-4094-9249-c0f87cce6981 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:04:55,100 [254] [DEBUG] [app] Ending request: urn:request:95b33c02-1ffb-4094-9249-c0f87cce6981 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:95b33c02-1ffb-4094-9249-c0f87cce6981', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:55,101 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.007 159 0.006) gunicorn-web stdout | 2025-11-04 09:04:55,101 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:04:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:55,103 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 4, 55, 102538), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 4, 55, 102574), 'quay', None, datetime.datetime(2025, 11, 3, 9, 4, 55, 102610), datetime.datetime(2025, 10, 28, 9, 4, 55, 102633), None]) gunicorn-web stdout | 2025-11-04 09:04:55,112 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:55,113 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:55,113 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:55,117 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:55,117 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:55,117 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:55,118 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:55,119 [257] [DEBUG] [app] Ending request: urn:request:290ea7f2-96e0-4a01-afce-9c870d815251 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:290ea7f2-96e0-4a01-afce-9c870d815251', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:55,120 [257] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.056 118 0.055) gunicorn-web stdout | 2025-11-04 09:04:55,120 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:04:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:04:55,122 [254] [DEBUG] [app] Starting request: urn:request:843d9322-aa62-468c-9090-29a8b2bccfd0 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:04:55,128 [254] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): localhost:8080 gunicorn-registry stdout | 2025-11-04 09:04:55,136 [250] [DEBUG] [app] Starting request: urn:request:3d8114ac-63d4-48aa-802f-5de57d0cea67 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:04:55,137 [250] [DEBUG] [app] Ending request: urn:request:3d8114ac-63d4-48aa-802f-5de57d0cea67 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3d8114ac-63d4-48aa-802f-5de57d0cea67', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:04:55,137 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:04:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:04:55,138 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:55,139 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:04:55,140 [257] [DEBUG] [app] Starting request: urn:request:6715ce0b-06e9-4ba2-a52a-92d72e9cbee6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:04:55,141 [257] [DEBUG] [app] Ending request: urn:request:6715ce0b-06e9-4ba2-a52a-92d72e9cbee6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6715ce0b-06e9-4ba2-a52a-92d72e9cbee6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:04:55,141 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:04:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:04:55,142 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:04:55,144 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 4, 55, 143153), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 4, 55, 143189), 'quay', None, datetime.datetime(2025, 11, 3, 9, 4, 55, 143216), datetime.datetime(2025, 10, 28, 9, 4, 55, 143237), None]) gunicorn-web stdout | 2025-11-04 09:04:55,153 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:04:55,154 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:04:55,154 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:04:55,158 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:04:55,158 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:04:55,158 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:04:55,159 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:04:55,160 [254] [DEBUG] [app] Ending request: urn:request:843d9322-aa62-468c-9090-29a8b2bccfd0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:843d9322-aa62-468c-9090-29a8b2bccfd0', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:04:55,160 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:04:55,160 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:04:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:04:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.039 118 0.039) notificationworker stdout | 2025-11-04 09:04:55,496 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:04:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:04:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:04:55 GMT)" (scheduled at 2025-11-04 09:04:55.661677+00:00) gcworker stdout | 2025-11-04 09:04:55,662 [66] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) gcworker stdout | 2025-11-04 09:04:55,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:07.663502+00:00 (in 12.000253 seconds) gcworker stdout | 2025-11-04 09:04:55,671 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037495670, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:04:55,674 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:04:55,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:04:55,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:04:55,872 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:04:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:04:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:07.766253+00:00 (in 9.999538 seconds) notificationworker stdout | 2025-11-04 09:04:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:07 GMT)" (scheduled at 2025-11-04 09:04:57.766253+00:00) notificationworker stdout | 2025-11-04 09:04:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:04:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 57, 767036), True, datetime.datetime(2025, 11, 4, 9, 4, 57, 767036), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:04:57,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:04:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:04:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:04:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:04:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:09.676299+00:00 (in 9.999454 seconds) proxycacheblobworker stdout | 2025-11-04 09:04:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:09 GMT)" (scheduled at 2025-11-04 09:04:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:04:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:04:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 4, 59, 677203), True, datetime.datetime(2025, 11, 4, 9, 4, 59, 677203), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:04:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:04:59,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:04:59,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:04:59,693 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:04:59,794 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:05:00,083 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:52.159714+00:00 (in 52.072797 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:00 GMT)" (scheduled at 2025-11-04 09:05:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:05:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:00 GMT)" executed successfully reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:05:00,891 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:05:01,090 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:19.706478+00:00 (in 17.998712 seconds) securityworker stdout | 2025-11-04 09:05:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:31 GMT)" (scheduled at 2025-11-04 09:05:01.707296+00:00) securityworker stdout | 2025-11-04 09:05:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:01,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:01,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:01,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:01,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:01,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:01,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:01,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 1, 713123), 1, 49]) securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:01,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:01,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 1, 713123), 1, 49]) securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:01,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:05:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:01,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:01,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:01,728 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:01,728 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:05:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:05:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:02 GMT)" (scheduled at 2025-11-04 09:05:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:05:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:05:02,083 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:32.081579+00:00 (in 29.998525 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:05:02,087 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:05:02,087 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:05:02,295 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:05:03,891 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:05:03,990 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:05:05,080 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:05:05,574 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:05:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:05:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:40.180734+00:00 (in 33.914343 seconds) repositorygcworker stdout | 2025-11-04 09:05:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:06 GMT)" (scheduled at 2025-11-04 09:05:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:05:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:05:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:05:06,570 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:05:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:06 GMT)" (scheduled at 2025-11-04 09:05:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:05:06,571 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:05:06,571 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:05:06,571 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:33.568037+00:00 (in 26.996460 seconds) autopruneworker stdout | 2025-11-04 09:05:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:05:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:07 GMT)" (scheduled at 2025-11-04 09:05:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:05:07,078 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:37.077358+00:00 (in 29.998988 seconds) autopruneworker stdout | 2025-11-04 09:05:07,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243507082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:05:07,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:05:07,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:05:07,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:05:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:25.661677+00:00 (in 17.997727 seconds) gcworker stdout | 2025-11-04 09:05:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:37 GMT)" (scheduled at 2025-11-04 09:05:07.663502+00:00) gcworker stdout | 2025-11-04 09:05:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:05:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246807670, None, 1, 0]) gcworker stdout | 2025-11-04 09:05:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:17.766253+00:00 (in 9.999278 seconds) notificationworker stdout | 2025-11-04 09:05:07,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:17 GMT)" (scheduled at 2025-11-04 09:05:07.766253+00:00) notificationworker stdout | 2025-11-04 09:05:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 7, 767303), True, datetime.datetime(2025, 11, 4, 9, 5, 7, 767303), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:05:07,771 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:05:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:07 GMT)" (scheduled at 2025-11-04 09:05:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:05:07,772 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:05:07,772 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:07.770322+00:00 (in 59.997514 seconds) notificationworker stdout | 2025-11-04 09:05:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:05:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:05:07,777 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:05:08,089 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:05:09,196 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:09,475 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:19.676299+00:00 (in 9.999528 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:19 GMT)" (scheduled at 2025-11-04 09:05:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 9, 677171), True, datetime.datetime(2025, 11, 4, 9, 5, 9, 677171), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:19 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:05:09,886 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:09,891 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:10,065 [254] [DEBUG] [app] Starting request: urn:request:724d16e7-427b-45d6-b7b7-7c9a69e430c3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:10,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:05:10,071 [260] [DEBUG] [app] Starting request: urn:request:ca9ea83c-e7d3-44b4-9cb7-0d90dcc5ca9d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:10,073 [260] [DEBUG] [app] Ending request: urn:request:ca9ea83c-e7d3-44b4-9cb7-0d90dcc5ca9d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ca9ea83c-e7d3-44b4-9cb7-0d90dcc5ca9d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.007 162 0.006) gunicorn-web stdout | 2025-11-04 09:05:10,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-registry stdout | 2025-11-04 09:05:10,074 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:10,075 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:10,076 [254] [DEBUG] [app] Starting request: urn:request:4213656b-504c-4355-8615-6d432ed6e0fc (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:10,076 [254] [DEBUG] [app] Ending request: urn:request:4213656b-504c-4355-8615-6d432ed6e0fc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4213656b-504c-4355-8615-6d432ed6e0fc', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:10,077 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:10,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:10,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:10,078 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:10,078 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:10,082 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:10,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:10,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:10,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:10,087 [254] [DEBUG] [app] Ending request: urn:request:724d16e7-427b-45d6-b7b7-7c9a69e430c3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:724d16e7-427b-45d6-b7b7-7c9a69e430c3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:10,087 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.023) gunicorn-web stdout | 2025-11-04 09:05:10,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:05:10,122 [254] [DEBUG] [app] Starting request: urn:request:788ccc0b-4023-4709-bc7e-d02eea9a3f7c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:05:10,124 [260] [DEBUG] [app] Starting request: urn:request:ff5ea43a-3507-42c1-bbb7-37f1fd5946f6 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:10,124 [260] [DEBUG] [app] Ending request: urn:request:ff5ea43a-3507-42c1-bbb7-37f1fd5946f6 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ff5ea43a-3507-42c1-bbb7-37f1fd5946f6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:10,124 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:10,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:10,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:10,131 [258] [DEBUG] [app] Starting request: urn:request:a32bfac1-e424-4477-9737-1ff9ed14954c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:10,132 [258] [DEBUG] [app] Ending request: urn:request:a32bfac1-e424-4477-9737-1ff9ed14954c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a32bfac1-e424-4477-9737-1ff9ed14954c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.007 159 0.007) gunicorn-web stdout | 2025-11-04 09:05:10,133 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:10,133 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:10,134 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:10,134 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:10,134 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:10,138 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:10,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:10,142 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:10,143 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:10,143 [254] [DEBUG] [app] Ending request: urn:request:788ccc0b-4023-4709-bc7e-d02eea9a3f7c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:788ccc0b-4023-4709-bc7e-d02eea9a3f7c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:10,144 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:10,144 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) quotaregistrysizeworker stdout | 2025-11-04 09:05:10,690 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:05:10,986 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:05:11,284 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:11,894 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:11,899 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:11,899 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:05:12,391 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:05:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:12 GMT)" (scheduled at 2025-11-04 09:05:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:05:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:05:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:12 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:05:12,483 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:38.480592+00:00 (in 25.997262 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:12,790 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:13,184 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:13,984 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:14,397 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:14,398 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:14,397 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:14,459 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:14,459 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:05:14,982 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:05:14,982 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:44.980431+00:00 (in 29.998007 seconds) buildlogsarchiver stdout | 2025-11-04 09:05:14,982 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:44 GMT)" (scheduled at 2025-11-04 09:05:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:05:14,983 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 5, 14, 982745), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:14,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:05:14,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:05:14,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:05:15,875 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,878 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,881 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,881 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,882 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,881 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,882 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,970 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:15,973 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:27.766253+00:00 (in 9.999566 seconds) notificationworker stdout | 2025-11-04 09:05:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:27 GMT)" (scheduled at 2025-11-04 09:05:17.766253+00:00) notificationworker stdout | 2025-11-04 09:05:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 17, 767007), True, datetime.datetime(2025, 11, 4, 9, 5, 17, 767007), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:17,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:17,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:05:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:29.676299+00:00 (in 9.999510 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:29 GMT)" (scheduled at 2025-11-04 09:05:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 19, 677121), True, datetime.datetime(2025, 11, 4, 9, 5, 19, 677121), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:05:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:31.707296+00:00 (in 12.000409 seconds) securityworker stdout | 2025-11-04 09:05:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:49 GMT)" (scheduled at 2025-11-04 09:05:19.706478+00:00) securityworker stdout | 2025-11-04 09:05:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:19,709 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:19,714 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:05:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:05:20,800 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:25,065 [257] [DEBUG] [app] Starting request: urn:request:da56c72f-566c-453d-b2a3-382f8f6dc511 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:25,067 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:05:25,069 [250] [DEBUG] [app] Starting request: urn:request:16413783-0d68-4d2e-92bd-ef741330bac7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:25,070 [250] [DEBUG] [app] Ending request: urn:request:16413783-0d68-4d2e-92bd-ef741330bac7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:16413783-0d68-4d2e-92bd-ef741330bac7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:25,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:25,070 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:05:25,071 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:25,072 [254] [DEBUG] [app] Starting request: urn:request:533de222-f19c-4547-bba5-eaaf76bf910d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:25,073 [254] [DEBUG] [app] Ending request: urn:request:533de222-f19c-4547-bba5-eaaf76bf910d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:533de222-f19c-4547-bba5-eaaf76bf910d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:25,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:25,073 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:25,074 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:25,074 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:25,074 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:25,079 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:25,079 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:25,083 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:25,084 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:25,085 [257] [DEBUG] [app] Ending request: urn:request:da56c72f-566c-453d-b2a3-382f8f6dc511 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:da56c72f-566c-453d-b2a3-382f8f6dc511', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:25,085 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:25,085 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.021) gunicorn-web stdout | 2025-11-04 09:05:25,123 [254] [DEBUG] [app] Starting request: urn:request:833c8ef4-097e-427d-9a6d-ad507f4ddac6 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:25,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:05:25,125 [250] [DEBUG] [app] Starting request: urn:request:781ad413-440c-41ed-b412-5066abd7c2c5 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:25,126 [250] [DEBUG] [app] Ending request: urn:request:781ad413-440c-41ed-b412-5066abd7c2c5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:781ad413-440c-41ed-b412-5066abd7c2c5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:25,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:25,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:05:25,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:25,128 [254] [DEBUG] [app] Starting request: urn:request:d2e1c372-554a-4666-b5da-41c312ed043f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:25,129 [254] [DEBUG] [app] Ending request: urn:request:d2e1c372-554a-4666-b5da-41c312ed043f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d2e1c372-554a-4666-b5da-41c312ed043f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:25,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:25,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:25,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:25,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:25,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:25,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:25,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:25,139 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:25,140 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:25,140 [254] [DEBUG] [app] Ending request: urn:request:833c8ef4-097e-427d-9a6d-ad507f4ddac6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:833c8ef4-097e-427d-9a6d-ad507f4ddac6', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:25,141 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:25,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.018) notificationworker stdout | 2025-11-04 09:05:25,510 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:37.663502+00:00 (in 12.001370 seconds) gcworker stdout | 2025-11-04 09:05:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:55 GMT)" (scheduled at 2025-11-04 09:05:25.661677+00:00) gcworker stdout | 2025-11-04 09:05:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037525662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:05:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:05:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:05:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:25,888 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:05:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:27.768179+00:00 (in 0.001475 seconds) notificationworker stdout | 2025-11-04 09:05:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:37 GMT)" (scheduled at 2025-11-04 09:05:27.766253+00:00) notificationworker stdout | 2025-11-04 09:05:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 27, 767036), True, datetime.datetime(2025, 11, 4, 9, 5, 27, 767036), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:27 GMT)" (scheduled at 2025-11-04 09:05:27.768179+00:00) notificationworker stdout | 2025-11-04 09:05:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:05:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:05:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:27,769 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:37.766253+00:00 (in 9.997224 seconds) notificationworker stdout | 2025-11-04 09:05:27,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:27,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:27,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:39.676299+00:00 (in 9.999610 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:39 GMT)" (scheduled at 2025-11-04 09:05:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 29, 676970), True, datetime.datetime(2025, 11, 4, 9, 5, 29, 676970), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:05:29,707 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:05:29,809 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:05:30,098 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:05:30,906 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:05:31,103 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:49.706478+00:00 (in 17.998758 seconds) securityworker stdout | 2025-11-04 09:05:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:01 GMT)" (scheduled at 2025-11-04 09:05:31.707296+00:00) securityworker stdout | 2025-11-04 09:05:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:31,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:31,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:31,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 31, 712171), 1, 49]) securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 31, 712171), 1, 49]) securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:31,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:05:31,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:05:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:05:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:05:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:05:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:05:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:05:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:02.081579+00:00 (in 29.999490 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:05:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:02 GMT)" (scheduled at 2025-11-04 09:05:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:05:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:05:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:05:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:05:32,308 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:05:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:53.560131+00:00 (in 19.997788 seconds) namespacegcworker stdout | 2025-11-04 09:05:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" (scheduled at 2025-11-04 09:05:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:05:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:05:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:05:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:05:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:06.569529+00:00 (in 33.001005 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:05:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" (scheduled at 2025-11-04 09:05:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:05:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:05:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 33, 568892), True, datetime.datetime(2025, 11, 4, 9, 5, 33, 568892), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:05:33,577 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:05:33,577 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:05:33,577 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:05:33,906 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:05:34,005 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:05:35,094 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:05:35,589 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:05:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:05:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:07.077358+00:00 (in 29.999448 seconds) autopruneworker stdout | 2025-11-04 09:05:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:07 GMT)" (scheduled at 2025-11-04 09:05:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:05:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243537082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:05:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:05:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:05:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:05:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:55.661677+00:00 (in 17.997733 seconds) gcworker stdout | 2025-11-04 09:05:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:07 GMT)" (scheduled at 2025-11-04 09:05:37.663502+00:00) gcworker stdout | 2025-11-04 09:05:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:05:37,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246837670, None, 1, 0]) gcworker stdout | 2025-11-04 09:05:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:47.766253+00:00 (in 9.999535 seconds) notificationworker stdout | 2025-11-04 09:05:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:47 GMT)" (scheduled at 2025-11-04 09:05:37.766253+00:00) notificationworker stdout | 2025-11-04 09:05:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 37, 767063), True, datetime.datetime(2025, 11, 4, 9, 5, 37, 767063), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:37,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:05:38,103 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:05:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:55.481710+00:00 (in 17.000598 seconds) exportactionlogsworker stdout | 2025-11-04 09:05:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:38 GMT)" (scheduled at 2025-11-04 09:05:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:05:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:05:38,483 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 38, 481535), True, datetime.datetime(2025, 11, 4, 9, 5, 38, 481535), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:05:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:05:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:05:38,491 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:05:39,211 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:05:39,489 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:47.679968+00:00 (in 8.003278 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:49 GMT)" (scheduled at 2025-11-04 09:05:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 39, 676961), True, datetime.datetime(2025, 11, 4, 9, 5, 39, 676961), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:49 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:05:39,901 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:05:39,906 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:40,066 [257] [DEBUG] [app] Starting request: urn:request:6e451622-47ca-4c0a-952d-6fde82d901a1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:40,067 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:05:40,072 [250] [DEBUG] [app] Starting request: urn:request:1efbf8ff-a87f-44a7-aa02-623a0459e452 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:40,073 [250] [DEBUG] [app] Ending request: urn:request:1efbf8ff-a87f-44a7-aa02-623a0459e452 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1efbf8ff-a87f-44a7-aa02-623a0459e452', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:05:40,073 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:40,073 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:40,074 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:40,075 [254] [DEBUG] [app] Starting request: urn:request:174493f7-b897-4d9d-8c61-6d141fcacd81 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:40,076 [254] [DEBUG] [app] Ending request: urn:request:174493f7-b897-4d9d-8c61-6d141fcacd81 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:174493f7-b897-4d9d-8c61-6d141fcacd81', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:40,076 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:40,076 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:40,077 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:40,077 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:40,077 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:40,081 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:40,081 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:40,085 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:40,087 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:40,087 [257] [DEBUG] [app] Ending request: urn:request:6e451622-47ca-4c0a-952d-6fde82d901a1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6e451622-47ca-4c0a-952d-6fde82d901a1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:40,087 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:40,088 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.022) gunicorn-web stdout | 2025-11-04 09:05:40,122 [254] [DEBUG] [app] Starting request: urn:request:5eedbd53-b07c-4d0a-9a04-eb7f63e3a972 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:40,123 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:05:40,125 [250] [DEBUG] [app] Starting request: urn:request:a0aa3ad4-a82e-431b-b491-fcc43f1f94ba (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:40,126 [250] [DEBUG] [app] Ending request: urn:request:a0aa3ad4-a82e-431b-b491-fcc43f1f94ba (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a0aa3ad4-a82e-431b-b491-fcc43f1f94ba', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:40,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:40,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:40,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:40,129 [258] [DEBUG] [app] Starting request: urn:request:9165d449-b138-4374-816f-08e25be078d1 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:40,130 [258] [DEBUG] [app] Ending request: urn:request:9165d449-b138-4374-816f-08e25be078d1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9165d449-b138-4374-816f-08e25be078d1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:40,130 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:40,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:40,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:40,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:40,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:40,134 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:40,134 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:40,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:40,140 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:40,140 [254] [DEBUG] [app] Ending request: urn:request:5eedbd53-b07c-4d0a-9a04-eb7f63e3a972 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5eedbd53-b07c-4d0a-9a04-eb7f63e3a972', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:40,140 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:40,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) repositorygcworker stdout | 2025-11-04 09:05:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:05:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:06.265965+00:00 (in 26.084785 seconds) repositorygcworker stdout | 2025-11-04 09:05:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:40 GMT)" (scheduled at 2025-11-04 09:05:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:05:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:05:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 40, 181522), True, datetime.datetime(2025, 11, 4, 9, 5, 40, 181522), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:05:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:05:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:05:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:05:40,705 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:05:41,001 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:05:41,299 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:41,912 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:41,917 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:41,918 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:05:42,406 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:42,805 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:43,198 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:05:44,001 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:44,415 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:44,417 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:44,417 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:44,477 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:05:44,477 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:05:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:05:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:14.980431+00:00 (in 29.999553 seconds) buildlogsarchiver stdout | 2025-11-04 09:05:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:14 GMT)" (scheduled at 2025-11-04 09:05:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:05:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 5, 44, 981538), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:05:44,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:05:44,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:05:44,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:05:45,901 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,979 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,987 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,987 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,989 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,990 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,991 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,998 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:05:45,999 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:49.676299+00:00 (in 1.995908 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:47 GMT)" (scheduled at 2025-11-04 09:05:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:05:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:57.766253+00:00 (in 9.999587 seconds) notificationworker stdout | 2025-11-04 09:05:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:57 GMT)" (scheduled at 2025-11-04 09:05:47.766253+00:00) notificationworker stdout | 2025-11-04 09:05:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 47, 766992), True, datetime.datetime(2025, 11, 4, 9, 5, 47, 766992), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:47,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:47,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:47,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:05:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:05:59.676299+00:00 (in 9.999505 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:59 GMT)" (scheduled at 2025-11-04 09:05:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 49, 677132), True, datetime.datetime(2025, 11, 4, 9, 5, 49, 677132), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:49,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:49,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:49,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:05:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:05:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:05:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:01.707296+00:00 (in 12.000404 seconds) securityworker stdout | 2025-11-04 09:05:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:19 GMT)" (scheduled at 2025-11-04 09:05:49.706478+00:00) securityworker stdout | 2025-11-04 09:05:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:05:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:05:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:05:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:05:49,715 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:49,716 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,717 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 22, 32]) securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stdout | 2025-11-04 09:05:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 36, 46]) securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 36-46 securityworker stdout | 2025-11-04 09:05:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-36 securityworker stdout | 2025-11-04 09:05:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 36-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-36 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:05:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with block 36-46 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-46 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 36-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-46 securityworker stdout | 2025-11-04 09:05:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 22-46 securityworker stdout | 2025-11-04 09:05:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:05:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:05:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 4, 14]) securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:05:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Right range 22-46 securityworker stdout | 2025-11-04 09:05:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-22 securityworker stdout | 2025-11-04 09:05:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-22 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:05:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 14, 24]) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:05:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:05:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-46 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 46 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 46-49 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 46-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:05:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stdout | 2025-11-04 09:05:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 46, 49]) securityworker stdout | 2025-11-04 09:05:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 46 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 46 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-46 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 49-46 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 29, 39]) securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,736 [92] [DEBUG] [util.migrate.allocator] Left range 29-39 securityworker stdout | 2025-11-04 09:05:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stdout | 2025-11-04 09:05:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 29-39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 39, 49]) securityworker stdout | 2025-11-04 09:05:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:05:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stdout | 2025-11-04 09:05:49,737 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 29-39 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Already merged with block 29-39 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-29 securityworker stdout | 2025-11-04 09:05:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 29-39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 29-39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stdout | 2025-11-04 09:05:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 16, 26]) securityworker stdout | 2025-11-04 09:05:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:05:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:05:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stdout | 2025-11-04 09:05:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,740 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stdout | 2025-11-04 09:05:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-16 securityworker stdout | 2025-11-04 09:05:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 6 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-16 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 6 securityworker stdout | 2025-11-04 09:05:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 6, 16]) securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 6-16 by worker securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 6-16 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 6-16 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 6-16 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 6-26 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Right range 6-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 6-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 6-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-6 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-6 securityworker stdout | 2025-11-04 09:05:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 1, 11]) securityworker stdout | 2025-11-04 09:05:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:05:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with block 6-26 securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 6-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 26-29 securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-29 securityworker stdout | 2025-11-04 09:05:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 26-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-29 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:05:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 26, 36]) securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 26 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 36 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 36-26 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:05:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 36 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 36-26 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 11, 21]) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:05:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:05:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:05:49,749 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stdout | 2025-11-04 09:05:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,750 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:05:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:05:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 37, 47]) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:05:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:05:49,752 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:05:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-37 securityworker stdout | 2025-11-04 09:05:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-37 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:05:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 25, 35]) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-37 securityworker stdout | 2025-11-04 09:05:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-37 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stdout | 2025-11-04 09:05:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 35, 45]) securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:05:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-47 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-47 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 25-47 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-25 securityworker stdout | 2025-11-04 09:05:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-25 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:05:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 21, 31]) securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-21 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-21 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-21 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-47 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-47 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-21 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 11-47 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stdout | 2025-11-04 09:05:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:05:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 1, 11]) securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-47 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:05:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 0, 49, 709642), 47, 49]) securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:05:49,760 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:05:49,761 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:05:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:05:50,814 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:00.086435+00:00 (in 7.925696 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:52 GMT)" (scheduled at 2025-11-04 09:05:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:05:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:05:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:05:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:33.561829+00:00 (in 40.001197 seconds) namespacegcworker stdout | 2025-11-04 09:05:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:53 GMT)" (scheduled at 2025-11-04 09:05:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:05:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:05:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 53, 560962), True, datetime.datetime(2025, 11, 4, 9, 5, 53, 560962), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:05:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:05:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:05:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:06:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:05:55,068 [261] [DEBUG] [app] Starting request: urn:request:30247b7d-7122-4411-97e6-b923104e55da (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:05:55,074 [261] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): localhost:8080 gunicorn-registry stdout | 2025-11-04 09:05:55,076 [250] [DEBUG] [app] Starting request: urn:request:6e96f1fb-dff5-4903-bbe1-9cf6c8ca5d9f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:55,076 [250] [DEBUG] [app] Ending request: urn:request:6e96f1fb-dff5-4903-bbe1-9cf6c8ca5d9f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6e96f1fb-dff5-4903-bbe1-9cf6c8ca5d9f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:05:55,077 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:55,077 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:55,078 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:55,080 [254] [DEBUG] [app] Starting request: urn:request:562b0997-9270-44bd-913f-96475f7c7e24 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:55,080 [254] [DEBUG] [app] Ending request: urn:request:562b0997-9270-44bd-913f-96475f7c7e24 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:562b0997-9270-44bd-913f-96475f7c7e24', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:55,080 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:05:55,081 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:55,086 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 5, 55, 82234), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 5, 55, 82273), 'quay', None, datetime.datetime(2025, 11, 3, 9, 5, 55, 82300), datetime.datetime(2025, 10, 28, 9, 5, 55, 84313), None]) gunicorn-web stdout | 2025-11-04 09:05:55,093 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:55,094 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:55,094 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:55,098 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:55,098 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:55,098 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:55,099 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:55,101 [261] [DEBUG] [app] Ending request: urn:request:30247b7d-7122-4411-97e6-b923104e55da (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:30247b7d-7122-4411-97e6-b923104e55da', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:55,101 [261] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.038 118 0.037) gunicorn-web stdout | 2025-11-04 09:05:55,102 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:05:55,122 [261] [DEBUG] [app] Starting request: urn:request:ff8826d4-6bf0-4a73-8e03-61c8b5e4df45 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:05:55,124 [250] [DEBUG] [app] Starting request: urn:request:9cf66944-fd5a-46e3-a764-09088703c5c9 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:05:55,124 [250] [DEBUG] [app] Ending request: urn:request:9cf66944-fd5a-46e3-a764-09088703c5c9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9cf66944-fd5a-46e3-a764-09088703c5c9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:05:55,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:05:55,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:05:55,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:05:55,127 [258] [DEBUG] [app] Starting request: urn:request:97cef5dc-5227-4489-a3c6-b866c7b5eccf (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:05:55,128 [258] [DEBUG] [app] Ending request: urn:request:97cef5dc-5227-4489-a3c6-b866c7b5eccf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:97cef5dc-5227-4489-a3c6-b866c7b5eccf', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:05:55,128 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:05:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:05:55,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:05:55,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:05:55,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:05:55,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:05:55,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:05:55,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:05:55,138 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:05:55,139 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:05:55,140 [261] [DEBUG] [app] Ending request: urn:request:ff8826d4-6bf0-4a73-8e03-61c8b5e4df45 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ff8826d4-6bf0-4a73-8e03-61c8b5e4df45', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:05:55,140 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:05:55,140 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:05:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:05:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) exportactionlogsworker stdout | 2025-11-04 09:05:55,481 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:05:55,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:12.482065+00:00 (in 16.999849 seconds) exportactionlogsworker stdout | 2025-11-04 09:05:55,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:10:55 GMT)" (scheduled at 2025-11-04 09:05:55.481710+00:00) exportactionlogsworker stdout | 2025-11-04 09:05:55,483 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:05:55,489 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), True, datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:05:55,491 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), True, datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 5, 55, 482543), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:05:55,491 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:05:55,492 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:10:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:05:55,523 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:05:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:05:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:07.663502+00:00 (in 12.001408 seconds) gcworker stdout | 2025-11-04 09:05:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:25 GMT)" (scheduled at 2025-11-04 09:05:55.661677+00:00) gcworker stdout | 2025-11-04 09:05:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037555662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:05:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:05:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:05:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:05:55,903 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:05:56,216 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:05:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:05:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.002) gunicorn-secscan stdout | 2025-11-04 09:05:56,589 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:05:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:05:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) notificationworker stdout | 2025-11-04 09:05:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:05:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:07.766253+00:00 (in 9.999583 seconds) notificationworker stdout | 2025-11-04 09:05:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:07 GMT)" (scheduled at 2025-11-04 09:05:57.766253+00:00) notificationworker stdout | 2025-11-04 09:05:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:05:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 57, 767065), True, datetime.datetime(2025, 11, 4, 9, 5, 57, 767065), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:05:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:05:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:05:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:05:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:05:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:09.676299+00:00 (in 9.999610 seconds) proxycacheblobworker stdout | 2025-11-04 09:05:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:09 GMT)" (scheduled at 2025-11-04 09:05:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:05:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:05:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 5, 59, 677243), True, datetime.datetime(2025, 11, 4, 9, 5, 59, 677243), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:05:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:05:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:05:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:05:59,721 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:05:59,823 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:52.159714+00:00 (in 52.072848 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:00 GMT)" (scheduled at 2025-11-04 09:06:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,093 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:06:00,093 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:00 GMT)" executed successfully builder stdout | 2025-11-04 09:06:00,112 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:06:00,920 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:06:01,117 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:19.706478+00:00 (in 17.998762 seconds) securityworker stdout | 2025-11-04 09:06:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:31 GMT)" (scheduled at 2025-11-04 09:06:01.707296+00:00) securityworker stdout | 2025-11-04 09:06:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:01,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:01,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 1, 713492), 1, 49]) securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 1, 713492), 1, 49]) securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:06:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:06:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:32.081579+00:00 (in 29.999563 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:06:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:32 GMT)" (scheduled at 2025-11-04 09:06:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:06:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:06:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:06:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:06:02,321 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:06:03,919 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:06:04,018 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:06:05,108 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:06:05,602 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:06:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:06:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:40.180734+00:00 (in 33.914357 seconds) repositorygcworker stdout | 2025-11-04 09:06:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:06 GMT)" (scheduled at 2025-11-04 09:06:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:06:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:06:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:06:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:06:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:33.568037+00:00 (in 26.998043 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:06:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:06 GMT)" (scheduled at 2025-11-04 09:06:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:06:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:06:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:06:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:06:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:37.077358+00:00 (in 29.999513 seconds) autopruneworker stdout | 2025-11-04 09:06:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:37 GMT)" (scheduled at 2025-11-04 09:06:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:06:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243567082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:06:07,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:06:07,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:06:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:06:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:25.661677+00:00 (in 17.997741 seconds) gcworker stdout | 2025-11-04 09:06:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:37 GMT)" (scheduled at 2025-11-04 09:06:07.663502+00:00) gcworker stdout | 2025-11-04 09:06:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:06:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246867670, None, 1, 0]) gcworker stdout | 2025-11-04 09:06:07,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:07,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:17.766253+00:00 (in 9.999484 seconds) notificationworker stdout | 2025-11-04 09:06:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:17 GMT)" (scheduled at 2025-11-04 09:06:07.766253+00:00) notificationworker stdout | 2025-11-04 09:06:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 7, 767108), True, datetime.datetime(2025, 11, 4, 9, 6, 7, 767108), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:06:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:06:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:07.770322+00:00 (in 59.999568 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:06:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:07 GMT)" (scheduled at 2025-11-04 09:06:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:06:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:06:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:06:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:06:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:06:08,116 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:06:09,224 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:09,502 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:19.676299+00:00 (in 9.999477 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:19 GMT)" (scheduled at 2025-11-04 09:06:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 9, 677197), True, datetime.datetime(2025, 11, 4, 9, 6, 9, 677197), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:19 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:06:09,915 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:09,919 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:10,065 [254] [DEBUG] [app] Starting request: urn:request:f5b17b6b-75c5-4245-821e-12ae909fc7be (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:06:10,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:06:10,069 [260] [DEBUG] [app] Starting request: urn:request:413148c5-65c2-411e-b2b6-2b2826b8eda3 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:10,070 [260] [DEBUG] [app] Ending request: urn:request:413148c5-65c2-411e-b2b6-2b2826b8eda3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:413148c5-65c2-411e-b2b6-2b2826b8eda3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:10,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.003) gunicorn-web stdout | 2025-11-04 09:06:10,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:10,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:10,072 [261] [DEBUG] [app] Starting request: urn:request:9e470040-7723-4b34-9790-d0fc964f57ac (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:10,073 [261] [DEBUG] [app] Ending request: urn:request:9e470040-7723-4b34-9790-d0fc964f57ac (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9e470040-7723-4b34-9790-d0fc964f57ac', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:06:10,073 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:10,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:10,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:10,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:10,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:10,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:10,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:10,083 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:10,084 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:10,085 [254] [DEBUG] [app] Ending request: urn:request:f5b17b6b-75c5-4245-821e-12ae909fc7be (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f5b17b6b-75c5-4245-821e-12ae909fc7be', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:10,085 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:10,085 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.021) gunicorn-web stdout | 2025-11-04 09:06:10,122 [254] [DEBUG] [app] Starting request: urn:request:4b1a2ded-e164-43d2-b35a-dae815b4bae8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:06:10,123 [250] [DEBUG] [app] Starting request: urn:request:407f963c-df8d-4b7d-95af-4fe09f0876aa (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:10,124 [250] [DEBUG] [app] Ending request: urn:request:407f963c-df8d-4b7d-95af-4fe09f0876aa (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:407f963c-df8d-4b7d-95af-4fe09f0876aa', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:10,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:10,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:10,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:10,126 [261] [DEBUG] [app] Starting request: urn:request:d6969433-3239-48aa-a1d4-0d0eafe00c3d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:10,127 [261] [DEBUG] [app] Ending request: urn:request:d6969433-3239-48aa-a1d4-0d0eafe00c3d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d6969433-3239-48aa-a1d4-0d0eafe00c3d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:10,127 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:10,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:10,127 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:10,127 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:10,127 [254] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:10,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:10,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:10,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:10,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:10,138 [254] [DEBUG] [app] Ending request: urn:request:4b1a2ded-e164-43d2-b35a-dae815b4bae8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4b1a2ded-e164-43d2-b35a-dae815b4bae8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:10,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:10,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.017) quotaregistrysizeworker stdout | 2025-11-04 09:06:10,719 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:06:11,015 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:06:11,313 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:11,927 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:11,931 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:11,931 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:06:12,420 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:06:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:38.480592+00:00 (in 25.998095 seconds) exportactionlogsworker stdout | 2025-11-04 09:06:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:12 GMT)" (scheduled at 2025-11-04 09:06:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:06:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:06:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:12 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:06:12,818 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:13,213 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:14,014 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:14,432 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:14,433 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:14,435 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:14,493 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:14,494 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:06:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:06:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:44.980431+00:00 (in 29.999501 seconds) buildlogsarchiver stdout | 2025-11-04 09:06:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:44 GMT)" (scheduled at 2025-11-04 09:06:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:06:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 6, 14, 981290), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:06:14,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:06:14,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:06:15,918 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:15,998 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,011 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,012 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,011 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,013 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,014 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,017 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:16,018 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:27.766253+00:00 (in 9.999577 seconds) notificationworker stdout | 2025-11-04 09:06:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:27 GMT)" (scheduled at 2025-11-04 09:06:17.766253+00:00) notificationworker stdout | 2025-11-04 09:06:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 17, 767059), True, datetime.datetime(2025, 11, 4, 9, 6, 17, 767059), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:17,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:17,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:17,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:06:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:29.676299+00:00 (in 9.999461 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:29 GMT)" (scheduled at 2025-11-04 09:06:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 19, 677111), True, datetime.datetime(2025, 11, 4, 9, 6, 19, 677111), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:06:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:31.707296+00:00 (in 12.000395 seconds) securityworker stdout | 2025-11-04 09:06:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:49 GMT)" (scheduled at 2025-11-04 09:06:19.706478+00:00) securityworker stdout | 2025-11-04 09:06:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:19,716 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:06:19,716 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:06:20,827 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:21,698 [254] [DEBUG] [app] Starting request: urn:request:8a63dbe3-dbc1-43ca-b5bf-f94cc57a66d8 (/api/v1/superuser/users/) {'X-Forwarded-For': '209.132.188.14, 10.131.0.44'} gunicorn-web stdout | 2025-11-04 09:06:21,698 [254] [DEBUG] [app] User loader loading deferred user with uuid: c065e551-0bcb-44ad-9a2d-bdaadda2b893 gunicorn-web stdout | 2025-11-04 09:06:21,699 [254] [DEBUG] [auth.cookie] Loading user from cookie: c065e551-0bcb-44ad-9a2d-bdaadda2b893 gunicorn-web stdout | 2025-11-04 09:06:21,699 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['c065e551-0bcb-44ad-9a2d-bdaadda2b893', False, 1, 0]) gunicorn-web stdout | 2025-11-04 09:06:21,709 [254] [DEBUG] [app] Ending request: urn:request:8a63dbe3-dbc1-43ca-b5bf-f94cc57a66d8 (/api/v1/superuser/users/) {'endpoint': 'api.superuserlist', 'request_id': 'urn:request:8a63dbe3-dbc1-43ca-b5bf-f94cc57a66d8', 'remote_addr': '10.131.0.44', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/api/v1/superuser/users/', 'path': '/api/v1/superuser/users/', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0'} gunicorn-web stdout | 2025-11-04 09:06:21,709 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:21,710 [254] [INFO] [gunicorn.access] 10.131.0.44 - - [04/Nov/2025:09:06:21 +0000] "GET /api/v1/superuser/users/ HTTP/1.0" 401 288 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/organization" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:06:21 +0000] "GET /api/v1/superuser/users/ HTTP/1.1" 401 288 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/organization" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) Gecko/20100101 Firefox/140.0" (0.014 1698 0.013) gunicorn-web stdout | 2025-11-04 09:06:25,066 [254] [DEBUG] [app] Starting request: urn:request:cb16e95a-9eda-4478-bde4-596f1eff8238 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:06:25,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:06:25,071 [250] [DEBUG] [app] Starting request: urn:request:8e0f760b-f99a-46ed-9fb3-b4f1ae7096f0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:25,071 [250] [DEBUG] [app] Ending request: urn:request:8e0f760b-f99a-46ed-9fb3-b4f1ae7096f0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8e0f760b-f99a-46ed-9fb3-b4f1ae7096f0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:06:25,072 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:25,072 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:25,073 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:25,075 [261] [DEBUG] [app] Starting request: urn:request:17fc4f3e-89ff-4bac-be6a-3cb076a9f64e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:25,075 [261] [DEBUG] [app] Ending request: urn:request:17fc4f3e-89ff-4bac-be6a-3cb076a9f64e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:17fc4f3e-89ff-4bac-be6a-3cb076a9f64e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:25,076 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:25,076 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:25,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:25,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:25,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:25,082 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:25,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:25,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:25,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:25,088 [254] [DEBUG] [app] Ending request: urn:request:cb16e95a-9eda-4478-bde4-596f1eff8238 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cb16e95a-9eda-4478-bde4-596f1eff8238', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:25,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:25,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.024) gunicorn-web stdout | 2025-11-04 09:06:25,122 [254] [DEBUG] [app] Starting request: urn:request:8dcfdf5c-7407-49f2-898f-5d47a5b9b5ae (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:06:25,123 [250] [DEBUG] [app] Starting request: urn:request:1b1e4fd2-a036-4f81-9797-095ad5c587a2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:25,124 [250] [DEBUG] [app] Ending request: urn:request:1b1e4fd2-a036-4f81-9797-095ad5c587a2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1b1e4fd2-a036-4f81-9797-095ad5c587a2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:25,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:25,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:25,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:25,127 [261] [DEBUG] [app] Starting request: urn:request:ab7ba178-008d-48ed-8455-f9dd8541e459 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:25,127 [261] [DEBUG] [app] Ending request: urn:request:ab7ba178-008d-48ed-8455-f9dd8541e459 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ab7ba178-008d-48ed-8455-f9dd8541e459', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:25,128 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:25,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:25,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:25,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:25,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:25,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:25,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:25,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:25,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:25,137 [254] [DEBUG] [app] Ending request: urn:request:8dcfdf5c-7407-49f2-898f-5d47a5b9b5ae (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8dcfdf5c-7407-49f2-898f-5d47a5b9b5ae', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:25,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:25,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.016) notificationworker stdout | 2025-11-04 09:06:25,537 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:37.663502+00:00 (in 12.001397 seconds) gcworker stdout | 2025-11-04 09:06:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:55 GMT)" (scheduled at 2025-11-04 09:06:25.661677+00:00) gcworker stdout | 2025-11-04 09:06:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037585662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:06:25,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:06:25,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:25,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:06:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:06:25,916 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:06:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:27.768179+00:00 (in 0.001442 seconds) notificationworker stdout | 2025-11-04 09:06:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:37 GMT)" (scheduled at 2025-11-04 09:06:27.766253+00:00) notificationworker stdout | 2025-11-04 09:06:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 27, 767149), True, datetime.datetime(2025, 11, 4, 9, 6, 27, 767149), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:37.766253+00:00 (in 9.997745 seconds) notificationworker stdout | 2025-11-04 09:06:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:27 GMT)" (scheduled at 2025-11-04 09:06:27.768179+00:00) notificationworker stdout | 2025-11-04 09:06:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:06:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:27,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:27,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:27,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:39.676299+00:00 (in 9.999585 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:39 GMT)" (scheduled at 2025-11-04 09:06:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 29, 677574), True, datetime.datetime(2025, 11, 4, 9, 6, 29, 677574), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:29,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:06:29,734 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:06:29,835 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:06:30,125 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:06:30,933 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:06:31,130 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:49.706478+00:00 (in 17.998747 seconds) securityworker stdout | 2025-11-04 09:06:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:01 GMT)" (scheduled at 2025-11-04 09:06:31.707296+00:00) securityworker stdout | 2025-11-04 09:06:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:31,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:31,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:31,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:31,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:31,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 31, 714616), 1, 49]) securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:31,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:31,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 31, 714616), 1, 49]) securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:06:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:31,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:06:31,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:06:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:06:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:02.081579+00:00 (in 29.999583 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:06:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:02 GMT)" (scheduled at 2025-11-04 09:06:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:06:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:06:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:06:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:06:32,334 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:06:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:53.560131+00:00 (in 19.997791 seconds) namespacegcworker stdout | 2025-11-04 09:06:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" (scheduled at 2025-11-04 09:06:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:06:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:06:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:06:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:06:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:06.569529+00:00 (in 33.001023 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:06:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" (scheduled at 2025-11-04 09:06:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:06:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:06:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 33, 568813), True, datetime.datetime(2025, 11, 4, 9, 6, 33, 568813), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:06:33,577 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:06:33,577 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:06:33,577 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:06:33,933 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:06:34,032 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:06:35,121 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:06:35,615 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:06:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:06:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:07.077358+00:00 (in 29.999561 seconds) autopruneworker stdout | 2025-11-04 09:06:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:07 GMT)" (scheduled at 2025-11-04 09:06:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:06:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243597082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:06:37,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:06:37,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:06:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:06:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:55.661677+00:00 (in 17.997763 seconds) gcworker stdout | 2025-11-04 09:06:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:07 GMT)" (scheduled at 2025-11-04 09:06:37.663502+00:00) gcworker stdout | 2025-11-04 09:06:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:06:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246897671, None, 1, 0]) gcworker stdout | 2025-11-04 09:06:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:47.766253+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:06:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:47 GMT)" (scheduled at 2025-11-04 09:06:37.766253+00:00) notificationworker stdout | 2025-11-04 09:06:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 37, 767062), True, datetime.datetime(2025, 11, 4, 9, 6, 37, 767062), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:06:38,129 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:06:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:12.482065+00:00 (in 34.001041 seconds) exportactionlogsworker stdout | 2025-11-04 09:06:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:38 GMT)" (scheduled at 2025-11-04 09:06:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:06:38,482 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:06:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 38, 482100), True, datetime.datetime(2025, 11, 4, 9, 6, 38, 482100), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:06:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:06:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:06:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:06:39,238 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:06:39,516 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:47.679968+00:00 (in 8.003234 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:49 GMT)" (scheduled at 2025-11-04 09:06:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 39, 677140), True, datetime.datetime(2025, 11, 4, 9, 6, 39, 677140), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:39,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:39,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:39,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:49 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:06:39,928 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:06:39,933 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:40,065 [254] [DEBUG] [app] Starting request: urn:request:5bb7c7cf-9f4a-4e29-b44f-40153e44403d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:06:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:06:40,068 [260] [DEBUG] [app] Starting request: urn:request:81226272-8de0-4981-b599-6be4693ad060 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:40,069 [260] [DEBUG] [app] Ending request: urn:request:81226272-8de0-4981-b599-6be4693ad060 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:81226272-8de0-4981-b599-6be4693ad060', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:40,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:06:40,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:40,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:40,073 [254] [DEBUG] [app] Starting request: urn:request:95d09398-da4d-4031-b45d-76cb50346d31 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:40,073 [254] [DEBUG] [app] Ending request: urn:request:95d09398-da4d-4031-b45d-76cb50346d31 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:95d09398-da4d-4031-b45d-76cb50346d31', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:40,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:40,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:40,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:40,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:40,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:40,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:40,083 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:40,085 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:40,085 [254] [DEBUG] [app] Ending request: urn:request:5bb7c7cf-9f4a-4e29-b44f-40153e44403d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5bb7c7cf-9f4a-4e29-b44f-40153e44403d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:40,085 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:40,086 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.021) gunicorn-web stdout | 2025-11-04 09:06:40,122 [261] [DEBUG] [app] Starting request: urn:request:e4199c98-42b6-4e63-b3ca-79f85cb87864 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:06:40,123 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:06:40,124 [260] [DEBUG] [app] Starting request: urn:request:1ca3574d-c149-45f3-8535-351ac45e438e (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:40,125 [260] [DEBUG] [app] Ending request: urn:request:1ca3574d-c149-45f3-8535-351ac45e438e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1ca3574d-c149-45f3-8535-351ac45e438e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:40,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:06:40,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:40,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:06:40,128 [254] [DEBUG] [app] Starting request: urn:request:095462cb-10de-4a7b-ad6d-b660087b736d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:40,128 [254] [DEBUG] [app] Ending request: urn:request:095462cb-10de-4a7b-ad6d-b660087b736d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:095462cb-10de-4a7b-ad6d-b660087b736d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:40,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:40,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:40,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:40,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:40,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:06:40,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:40,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:06:40,136 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:06:40,137 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:40,139 [261] [DEBUG] [app] Ending request: urn:request:e4199c98-42b6-4e63-b3ca-79f85cb87864 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e4199c98-42b6-4e63-b3ca-79f85cb87864', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:40,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:40,139 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) repositorygcworker stdout | 2025-11-04 09:06:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:06:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:06.265965+00:00 (in 26.084783 seconds) repositorygcworker stdout | 2025-11-04 09:06:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:40 GMT)" (scheduled at 2025-11-04 09:06:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:06:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:06:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 40, 181520), True, datetime.datetime(2025, 11, 4, 9, 6, 40, 181520), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:06:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:06:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:06:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:06:40,733 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:06:41,028 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:06:41,326 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:41,941 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:41,945 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:06:41,945 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:06:42,433 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:42,833 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:06:43,226 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:44,028 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:44,450 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:44,450 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:44,450 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:44,508 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:06:44,509 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:06:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:06:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:14.980431+00:00 (in 29.999538 seconds) buildlogsarchiver stdout | 2025-11-04 09:06:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:14 GMT)" (scheduled at 2025-11-04 09:06:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:06:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 6, 44, 981174), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:06:44,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:06:44,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:06:44,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:06:45,934 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,017 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,030 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,031 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,034 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,035 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,035 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,038 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:06:46,039 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:49.676299+00:00 (in 1.995875 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:47 GMT)" (scheduled at 2025-11-04 09:06:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:06:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:57.766253+00:00 (in 9.999579 seconds) notificationworker stdout | 2025-11-04 09:06:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:57 GMT)" (scheduled at 2025-11-04 09:06:47.766253+00:00) notificationworker stdout | 2025-11-04 09:06:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 47, 767076), True, datetime.datetime(2025, 11, 4, 9, 6, 47, 767076), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:47,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:47,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:47,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:06:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:06:59.676299+00:00 (in 9.999538 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:59 GMT)" (scheduled at 2025-11-04 09:06:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 49, 677085), True, datetime.datetime(2025, 11, 4, 9, 6, 49, 677085), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:06:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:06:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:06:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:01.707296+00:00 (in 12.000366 seconds) securityworker stdout | 2025-11-04 09:06:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:19 GMT)" (scheduled at 2025-11-04 09:06:49.706478+00:00) securityworker stdout | 2025-11-04 09:06:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:06:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:06:49,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:06:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:49,716 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:06:49,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 2, 12]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:06:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-49 securityworker stdout | 2025-11-04 09:06:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 13, 23]) securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Right range 13-23 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-13 securityworker stdout | 2025-11-04 09:06:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 13-23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-13 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:06:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 12, 22]) securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-23 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 23-49 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-49 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-49 securityworker stdout | 2025-11-04 09:06:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 34, 44]) securityworker stdout | 2025-11-04 09:06:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:06:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:06:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:06:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 23-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [util.migrate.allocator] Left range 34-44 securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Left range 34-44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:06:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 44, 49]) securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 34-44 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Already merged with block 34-44 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 34 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 34-44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 34-44 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 34 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-34 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 23-34 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-34 securityworker stdout | 2025-11-04 09:06:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-34 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:06:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 24, 34]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 24 securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 23-24 securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-24 securityworker stdout | 2025-11-04 09:06:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 24 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-24 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-24 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:06:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 23, 33]) securityworker stdout | 2025-11-04 09:06:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:06:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 33 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 33 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 33-23 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 33-23 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 18, 28]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stdout | 2025-11-04 09:06:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 1, 11]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-18 securityworker stdout | 2025-11-04 09:06:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-18 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:06:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 11, 21]) securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 35, 45]) securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stdout | 2025-11-04 09:06:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,741 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stdout | 2025-11-04 09:06:49,741 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:06:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 45, 49]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 28-35 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 28-35 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-35 securityworker stdout | 2025-11-04 09:06:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-35 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:06:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 28, 38]) securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 28 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 38 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 38-28 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:06:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 38 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 38-28 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 21, 31]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:06:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:06:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 11, 21]) securityworker stdout | 2025-11-04 09:06:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:06:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-31 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Right range 11-31 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:06:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 1, 11]) securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-31 securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 31-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stdout | 2025-11-04 09:06:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:06:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 38, 48]) securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 31-49 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:06:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 48, 49]) securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 31-38 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-38 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-38 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-38 securityworker stdout | 2025-11-04 09:06:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:06:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 1, 49, 710487), 31, 41]) securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:06:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:06:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:49,755 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stderr | 2025-11-04 09:06:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:06:49,755 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:06:50,840 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:00.086435+00:00 (in 7.926288 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:52 GMT)" (scheduled at 2025-11-04 09:06:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,160 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:06:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:06:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:06:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:32.561493+00:00 (in 39.000787 seconds) namespacegcworker stdout | 2025-11-04 09:06:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:53 GMT)" (scheduled at 2025-11-04 09:06:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:06:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:06:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 53, 561042), True, datetime.datetime(2025, 11, 4, 9, 6, 53, 561042), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,570 [75] [DEBUG] [peewee] ('UPDATE "queueitem" SET "available" = %s, "processing_expires" = %s, "retries_remaining" = ("queueitem"."retries_remaining" - %s), "state_id" = %s WHERE (("queueitem"."id" = %s) AND ("queueitem"."state_id" = %s))', [False, datetime.datetime(2025, 11, 4, 12, 6, 53, 561042), 1, 'ce683e5d-07a1-42dc-bccb-76a812531d1b', 2, '38a9df49-4d32-416e-901c-c63402820458']) namespacegcworker stdout | 2025-11-04 09:06:53,571 [75] [DEBUG] [workers.queueworker] Queue gave us some work: {"marker_id": 2, "original_username": "testorg"} namespacegcworker stdout | 2025-11-04 09:06:53,572 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:06:53,572 [75] [DEBUG] [util.locking] Acquiring global lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:53,572 [75] [DEBUG] [redis_lock.acquire] Getting 'lock:LARGE_GARBAGE_COLLECTION' ... namespacegcworker stdout | 2025-11-04 09:06:53,577 [75] [INFO] [redis_lock.acquire] Got lock for 'lock:LARGE_GARBAGE_COLLECTION'. namespacegcworker stdout | 2025-11-04 09:06:53,577 [75] [DEBUG] [util.locking] Acquired lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:53,577 [75] [DEBUG] [__main__] Got namespace GC queue item: {'marker_id': 2, 'original_username': 'testorg'} namespacegcworker stdout | 2025-11-04 09:06:53,577 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."marked", "t1"."original_username", "t1"."original_email", "t1"."queue_id" FROM "deletednamespace" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,583 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,585 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['dockerfilebuild/95595bdd-7ef2-4730-9b0a-15a675d82fb0/%']) namespacegcworker stdout | 2025-11-04 09:06:53,586 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['notification/95595bdd-7ef2-4730-9b0a-15a675d82fb0/%']) namespacegcworker stdout | 2025-11-04 09:06:53,587 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['repositorygc/95595bdd-7ef2-4730-9b0a-15a675d82fb0/%']) namespacegcworker stdout | 2025-11-04 09:06:53,587 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."namespace_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,588 [75] [DEBUG] [peewee] ('UPDATE "repository" SET "namespace_user_id" = %s, "name" = %s, "visibility_id" = %s, "description" = %s, "badge_token" = %s, "kind_id" = %s, "trust_enabled" = %s, "state" = %s WHERE ("repository"."id" = %s)', [3, 'alpine', 2, None, '97503d1a-b4e1-47ae-a65e-4e982365a5bc', 1, False, 3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,592 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,593 [75] [DEBUG] [data.model.gc] Purging repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,593 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:53,595 [75] [DEBUG] [data.model.gc] Found 10 tags to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,595 [75] [DEBUG] [data.model.gc] Deleting tag 2 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,599 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,599 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-87fbf45b-1ee6-47c8-b597-168061a86a2e']) namespacegcworker stdout | 2025-11-04 09:06:53,600 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,601 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-87fbf45b-1ee6-47c8-b597-168061a86a2e namespacegcworker stdout | 2025-11-04 09:06:53,601 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,603 [75] [DEBUG] [data.model.gc] Deleting tag 3 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,603 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,604 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-54cac4c0-2e79-4cc7-b71d-97193eda49ab']) namespacegcworker stdout | 2025-11-04 09:06:53,604 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,605 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-54cac4c0-2e79-4cc7-b71d-97193eda49ab namespacegcworker stdout | 2025-11-04 09:06:53,605 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,607 [75] [DEBUG] [data.model.gc] Deleting tag 4 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,607 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,608 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-c3f19ae7-eee1-42a9-bd12-bc20a16da421']) namespacegcworker stdout | 2025-11-04 09:06:53,608 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,609 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-c3f19ae7-eee1-42a9-bd12-bc20a16da421 namespacegcworker stdout | 2025-11-04 09:06:53,609 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,610 [75] [DEBUG] [data.model.gc] Deleting tag 5 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,611 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,611 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-96f1f098-1ab4-4baa-bb2e-4193d3767983']) namespacegcworker stdout | 2025-11-04 09:06:53,612 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,612 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-96f1f098-1ab4-4baa-bb2e-4193d3767983 namespacegcworker stdout | 2025-11-04 09:06:53,612 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,613 [75] [DEBUG] [data.model.gc] Deleting tag 6 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,614 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,614 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-45d0addd-5936-446d-bbdd-f9a0d98e41ab']) namespacegcworker stdout | 2025-11-04 09:06:53,615 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,615 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-45d0addd-5936-446d-bbdd-f9a0d98e41ab namespacegcworker stdout | 2025-11-04 09:06:53,615 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,616 [75] [DEBUG] [data.model.gc] Deleting tag 7 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,617 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,617 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-c228c420-f2af-4882-8ad3-d6b3caf5a43e']) namespacegcworker stdout | 2025-11-04 09:06:53,618 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,618 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-c228c420-f2af-4882-8ad3-d6b3caf5a43e namespacegcworker stdout | 2025-11-04 09:06:53,618 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,620 [75] [DEBUG] [data.model.gc] Deleting tag 8 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,620 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,621 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-d6e4ec25-9a53-4d40-9a10-b4df6c8c7360']) namespacegcworker stdout | 2025-11-04 09:06:53,621 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,621 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-d6e4ec25-9a53-4d40-9a10-b4df6c8c7360 namespacegcworker stdout | 2025-11-04 09:06:53,621 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,623 [75] [DEBUG] [data.model.gc] Deleting tag 9 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,623 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,624 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-7c985836-c54e-4a29-8d8e-5d1541d4753d']) namespacegcworker stdout | 2025-11-04 09:06:53,624 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,624 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-7c985836-c54e-4a29-8d8e-5d1541d4753d namespacegcworker stdout | 2025-11-04 09:06:53,624 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,626 [75] [DEBUG] [data.model.gc] Deleting tag 10 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,626 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,627 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-1668c948-7136-4062-aade-a572dd5107a2']) namespacegcworker stdout | 2025-11-04 09:06:53,627 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,627 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-1668c948-7136-4062-aade-a572dd5107a2 namespacegcworker stdout | 2025-11-04 09:06:53,627 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,629 [75] [DEBUG] [data.model.gc] Deleting tag 11 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,629 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,630 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-dc71435a-4be4-429d-9cad-a1bd093a5ddb']) namespacegcworker stdout | 2025-11-04 09:06:53,630 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,630 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-dc71435a-4be4-429d-9cad-a1bd093a5ddb namespacegcworker stdout | 2025-11-04 09:06:53,631 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,632 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,633 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,634 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,635 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,636 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,637 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,637 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,638 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,639 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,640 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,640 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,641 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,642 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,643 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,643 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,644 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,645 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,646 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,646 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,647 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,648 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:53,648 [75] [DEBUG] [data.model.gc] Found 7 tags to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,648 [75] [DEBUG] [data.model.gc] Deleting tag 12 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,649 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,649 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-36d395ec-99c3-4d05-a283-6e93606c8f43']) namespacegcworker stdout | 2025-11-04 09:06:53,650 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,650 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-36d395ec-99c3-4d05-a283-6e93606c8f43 namespacegcworker stdout | 2025-11-04 09:06:53,650 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,651 [75] [DEBUG] [data.model.gc] Deleting tag 13 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,652 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,652 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-ebd73a38-84e6-41c8-bd06-d05c2b69a387']) namespacegcworker stdout | 2025-11-04 09:06:53,653 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:53,653 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-ebd73a38-84e6-41c8-bd06-d05c2b69a387 namespacegcworker stdout | 2025-11-04 09:06:53,653 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:53,655 [75] [DEBUG] [data.model.gc] Deleting tag 14 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,656 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,656 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-17295a98-c21c-4f36-903d-4ff6d05cce1d']) namespacegcworker stdout | 2025-11-04 09:06:53,657 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:53,657 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-17295a98-c21c-4f36-903d-4ff6d05cce1d namespacegcworker stdout | 2025-11-04 09:06:53,657 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:53,659 [75] [DEBUG] [data.model.gc] Deleting tag 15 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,659 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,660 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-8c54284e-57dd-49a2-b449-7ea96b6bfba3']) namespacegcworker stdout | 2025-11-04 09:06:53,660 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:53,660 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-8c54284e-57dd-49a2-b449-7ea96b6bfba3 namespacegcworker stdout | 2025-11-04 09:06:53,660 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:53,662 [75] [DEBUG] [data.model.gc] Deleting tag 16 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,662 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,663 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-758ed318-0273-4ada-ab2e-ff1aaec12a84']) namespacegcworker stdout | 2025-11-04 09:06:53,663 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:53,664 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-758ed318-0273-4ada-ab2e-ff1aaec12a84 namespacegcworker stdout | 2025-11-04 09:06:53,664 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:53,665 [75] [DEBUG] [data.model.gc] Deleting tag 17 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,665 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,666 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, '$temp-684e5047-bab5-44a0-be6c-b376e3587179']) namespacegcworker stdout | 2025-11-04 09:06:53,667 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:53,667 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-684e5047-bab5-44a0-be6c-b376e3587179 namespacegcworker stdout | 2025-11-04 09:06:53,667 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:53,668 [75] [DEBUG] [data.model.gc] Deleting tag 18 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:53,669 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,670 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [7, 'alpine1']) namespacegcworker stdout | 2025-11-04 09:06:53,670 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,670 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: alpine1 namespacegcworker stdout | 2025-11-04 09:06:53,670 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,672 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,673 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,673 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,674 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,675 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,675 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,676 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,677 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,678 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,678 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,679 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,680 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,681 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,682 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,683 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,685 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,686 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,687 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,687 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,688 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,689 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,690 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,690 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [18, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,691 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [18, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,692 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [18, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,692 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [18, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,693 [75] [DEBUG] [data.model.quota] no blobs found for manifest 18 in repository 7, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,693 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [18, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,693 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715']) namespacegcworker stdout | 2025-11-04 09:06:53,694 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:53,697 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,698 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,698 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715 namespacegcworker stdout | 2025-11-04 09:06:53,701 [75] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 namespacegcworker stdout | 2025-11-04 09:06:53,715 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,716 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,717 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,717 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,718 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,720 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,720 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,721 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,721 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,722 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,723 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,723 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,724 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [2, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,725 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [2, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,725 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [2, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,726 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [2, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,726 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,727 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,727 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,727 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [2, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,728 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:08001109a7d679fe33b04fa51d681bd40b975d8f5cea8c3ef6c0eccb6a7338ce']) namespacegcworker stdout | 2025-11-04 09:06:53,728 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:53,731 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:08001109a7d679fe33b04fa51d681bd40b975d8f5cea8c3ef6c0eccb6a7338ce', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,732 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,732 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:08001109a7d679fe33b04fa51d681bd40b975d8f5cea8c3ef6c0eccb6a7338ce namespacegcworker stdout | 2025-11-04 09:06:53,750 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:08001109a7d679fe33b04fa51d681bd40b975d8f5cea8c3ef6c0eccb6a7338ce HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,751 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:08001109a7d679fe33b04fa51d681bd40b975d8f5cea8c3ef6c0eccb6a7338ce from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,751 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,752 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,753 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,754 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,755 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,755 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,756 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,757 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,757 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,758 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,759 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,759 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,760 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,760 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,761 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,761 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,762 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,762 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [3, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,762 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:bd4199eb785a74df05d995ccf017888413746b5566e0b401b93a173859f0827f']) namespacegcworker stdout | 2025-11-04 09:06:53,762 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:53,764 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:bd4199eb785a74df05d995ccf017888413746b5566e0b401b93a173859f0827f', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,765 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,765 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:bd4199eb785a74df05d995ccf017888413746b5566e0b401b93a173859f0827f namespacegcworker stdout | 2025-11-04 09:06:53,770 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:bd4199eb785a74df05d995ccf017888413746b5566e0b401b93a173859f0827f HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,771 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:bd4199eb785a74df05d995ccf017888413746b5566e0b401b93a173859f0827f from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,771 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,772 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,773 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,774 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,774 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,775 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,776 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,777 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,777 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,778 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,779 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [4, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,780 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [4, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,780 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [4, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,781 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [4, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,782 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,782 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,782 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,782 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [4, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,783 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:a1c858aaff07c8da76e480d63c94e025c6317862dd2825c5ab2f4bf138a4c5dd']) namespacegcworker stdout | 2025-11-04 09:06:53,783 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:53,785 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:a1c858aaff07c8da76e480d63c94e025c6317862dd2825c5ab2f4bf138a4c5dd', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,786 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,786 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:a1c858aaff07c8da76e480d63c94e025c6317862dd2825c5ab2f4bf138a4c5dd namespacegcworker stdout | 2025-11-04 09:06:53,798 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:a1c858aaff07c8da76e480d63c94e025c6317862dd2825c5ab2f4bf138a4c5dd HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,798 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:a1c858aaff07c8da76e480d63c94e025c6317862dd2825c5ab2f4bf138a4c5dd from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,799 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,799 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,800 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,801 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,802 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,803 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,804 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,804 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,805 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,806 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,807 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [5, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,807 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [5, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,808 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [5, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,808 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [5, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,809 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,809 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,809 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,810 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [5, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,810 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:1475ea648c2027063f90bc98072ee2197ceec56e17de472f7f9777b8944e9c38']) namespacegcworker stdout | 2025-11-04 09:06:53,810 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:53,812 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:1475ea648c2027063f90bc98072ee2197ceec56e17de472f7f9777b8944e9c38', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,813 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,813 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:1475ea648c2027063f90bc98072ee2197ceec56e17de472f7f9777b8944e9c38 namespacegcworker stdout | 2025-11-04 09:06:53,824 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:1475ea648c2027063f90bc98072ee2197ceec56e17de472f7f9777b8944e9c38 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,824 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:1475ea648c2027063f90bc98072ee2197ceec56e17de472f7f9777b8944e9c38 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,824 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,826 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,826 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,827 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,828 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,829 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,830 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,830 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,831 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,832 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,833 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [6, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,833 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [6, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,834 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [6, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,834 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [6, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,835 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,835 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,836 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,836 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [6, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,836 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:0b762e487f7a8eb308da049b9a1edfaef7345f847399c27899d49c10e9c325b9']) namespacegcworker stdout | 2025-11-04 09:06:53,836 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:53,838 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:0b762e487f7a8eb308da049b9a1edfaef7345f847399c27899d49c10e9c325b9', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,839 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,839 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:0b762e487f7a8eb308da049b9a1edfaef7345f847399c27899d49c10e9c325b9 namespacegcworker stdout | 2025-11-04 09:06:53,851 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:0b762e487f7a8eb308da049b9a1edfaef7345f847399c27899d49c10e9c325b9 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,852 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:0b762e487f7a8eb308da049b9a1edfaef7345f847399c27899d49c10e9c325b9 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,852 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,853 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,854 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,855 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,855 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,856 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,857 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,858 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,858 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,859 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,860 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [7, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,860 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [7, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,861 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [7, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,861 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [7, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,862 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,862 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,863 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,863 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [7, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,863 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:256ffbe4e66db68c5d3ecd54b64e5b341158a10659859a838647d1432ea6a30a']) namespacegcworker stdout | 2025-11-04 09:06:53,864 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:53,866 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:256ffbe4e66db68c5d3ecd54b64e5b341158a10659859a838647d1432ea6a30a', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,866 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,866 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:256ffbe4e66db68c5d3ecd54b64e5b341158a10659859a838647d1432ea6a30a namespacegcworker stdout | 2025-11-04 09:06:53,871 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:256ffbe4e66db68c5d3ecd54b64e5b341158a10659859a838647d1432ea6a30a HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,871 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:256ffbe4e66db68c5d3ecd54b64e5b341158a10659859a838647d1432ea6a30a from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,872 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,873 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,874 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,875 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,876 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,876 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,877 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,878 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,879 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,880 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,881 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [8, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,881 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [8, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,882 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [8, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,882 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [8, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,883 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,883 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,883 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,883 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [8, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,884 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:008448246686fe28544e36ba89e7bc7fbe6dad8a2ccdef576875c74298c46348']) namespacegcworker stdout | 2025-11-04 09:06:53,884 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:53,886 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:008448246686fe28544e36ba89e7bc7fbe6dad8a2ccdef576875c74298c46348', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,887 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,887 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:008448246686fe28544e36ba89e7bc7fbe6dad8a2ccdef576875c74298c46348 namespacegcworker stdout | 2025-11-04 09:06:53,899 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:008448246686fe28544e36ba89e7bc7fbe6dad8a2ccdef576875c74298c46348 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,899 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:008448246686fe28544e36ba89e7bc7fbe6dad8a2ccdef576875c74298c46348 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,900 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,901 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,902 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,903 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,903 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,904 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,904 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,905 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,906 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,907 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,908 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [9, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,908 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [9, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,908 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [9, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,909 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [9, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,910 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,910 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,910 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,911 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [9, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,911 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:c2f35176d51645644dac998abfd7357827f01ca6d7a66b9c846178ddec8e3b80']) namespacegcworker stdout | 2025-11-04 09:06:53,911 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:53,913 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:c2f35176d51645644dac998abfd7357827f01ca6d7a66b9c846178ddec8e3b80', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,914 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,914 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:c2f35176d51645644dac998abfd7357827f01ca6d7a66b9c846178ddec8e3b80 namespacegcworker stdout | 2025-11-04 09:06:53,919 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:c2f35176d51645644dac998abfd7357827f01ca6d7a66b9c846178ddec8e3b80 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,920 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:c2f35176d51645644dac998abfd7357827f01ca6d7a66b9c846178ddec8e3b80 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,920 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,921 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,922 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,923 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,924 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,924 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,925 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,926 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,927 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,927 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,929 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [10, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,929 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [10, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,929 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [10, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,930 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [10, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,930 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,931 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,931 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,931 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [10, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,932 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:af11d8e82a84ada96b5af358e796120e1c83f685d077b25ec05af8e6bc8827b6']) namespacegcworker stdout | 2025-11-04 09:06:53,932 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:53,934 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:af11d8e82a84ada96b5af358e796120e1c83f685d077b25ec05af8e6bc8827b6', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,934 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,934 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:af11d8e82a84ada96b5af358e796120e1c83f685d077b25ec05af8e6bc8827b6 namespacegcworker stdout | 2025-11-04 09:06:53,946 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:af11d8e82a84ada96b5af358e796120e1c83f685d077b25ec05af8e6bc8827b6 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,946 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:af11d8e82a84ada96b5af358e796120e1c83f685d077b25ec05af8e6bc8827b6 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,946 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,947 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,948 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,949 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,950 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,950 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,951 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,951 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,952 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,953 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,954 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [11, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,955 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [11, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,955 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [11, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,956 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [11, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,956 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,956 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,957 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,957 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [11, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,957 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:c14851cd48d13b30981b7513c11e72f2cd7a90217efbce4d48233b5cfe23182f']) namespacegcworker stdout | 2025-11-04 09:06:53,957 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:53,959 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:c14851cd48d13b30981b7513c11e72f2cd7a90217efbce4d48233b5cfe23182f', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,960 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,961 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:c14851cd48d13b30981b7513c11e72f2cd7a90217efbce4d48233b5cfe23182f namespacegcworker stdout | 2025-11-04 09:06:53,966 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:c14851cd48d13b30981b7513c11e72f2cd7a90217efbce4d48233b5cfe23182f HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,966 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:c14851cd48d13b30981b7513c11e72f2cd7a90217efbce4d48233b5cfe23182f from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,967 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,968 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,969 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,970 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,971 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,972 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,973 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,974 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,975 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,976 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,977 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [12, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,977 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [12, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,978 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [12, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,979 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [12, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,980 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,980 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:53,981 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:53,981 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [12, 7]) namespacegcworker stdout | 2025-11-04 09:06:53,981 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:b058f50906dff72a20c83582eddad899cd501b7e4a0caca511cfdad19a129c30']) namespacegcworker stdout | 2025-11-04 09:06:53,982 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:53,984 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:b058f50906dff72a20c83582eddad899cd501b7e4a0caca511cfdad19a129c30', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,985 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:53,985 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:b058f50906dff72a20c83582eddad899cd501b7e4a0caca511cfdad19a129c30 namespacegcworker stdout | 2025-11-04 09:06:53,997 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:b058f50906dff72a20c83582eddad899cd501b7e4a0caca511cfdad19a129c30 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:53,997 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:b058f50906dff72a20c83582eddad899cd501b7e4a0caca511cfdad19a129c30 from secscan service namespacegcworker stdout | 2025-11-04 09:06:53,998 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:53,999 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,000 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,001 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,002 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,004 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,004 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,005 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,006 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,007 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,008 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [13, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,008 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [13, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,009 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [13, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,010 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [13, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,010 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,011 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:54,011 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:54,012 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [13, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,012 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:cb1862fbe7951d7e19bf6012f18368a99531938db355638492374c14c50cd50d']) namespacegcworker stdout | 2025-11-04 09:06:54,013 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,015 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:cb1862fbe7951d7e19bf6012f18368a99531938db355638492374c14c50cd50d', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,016 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:54,016 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:cb1862fbe7951d7e19bf6012f18368a99531938db355638492374c14c50cd50d namespacegcworker stdout | 2025-11-04 09:06:54,021 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:cb1862fbe7951d7e19bf6012f18368a99531938db355638492374c14c50cd50d HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,021 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:cb1862fbe7951d7e19bf6012f18368a99531938db355638492374c14c50cd50d from secscan service namespacegcworker stdout | 2025-11-04 09:06:54,022 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,022 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,023 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,026 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,027 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,027 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,028 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,029 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,029 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,031 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,032 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [14, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,033 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [14, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,033 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [14, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,034 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [14, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,034 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,035 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:54,035 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:54,035 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [14, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,036 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:d17c97e1ccf6490cb90fa17f6f021e404401d2656928631c73fd75ee711d87bd']) namespacegcworker stdout | 2025-11-04 09:06:54,036 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,038 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:d17c97e1ccf6490cb90fa17f6f021e404401d2656928631c73fd75ee711d87bd', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,039 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:54,039 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:d17c97e1ccf6490cb90fa17f6f021e404401d2656928631c73fd75ee711d87bd namespacegcworker stdout | 2025-11-04 09:06:54,050 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:d17c97e1ccf6490cb90fa17f6f021e404401d2656928631c73fd75ee711d87bd HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,051 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:d17c97e1ccf6490cb90fa17f6f021e404401d2656928631c73fd75ee711d87bd from secscan service namespacegcworker stdout | 2025-11-04 09:06:54,051 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,052 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,053 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,054 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,054 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,055 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,056 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,057 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,057 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,058 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,059 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [15, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,059 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [15, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,060 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [15, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,061 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [15, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,061 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,062 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:54,062 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:54,063 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [15, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,064 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:6c177c7cb8fd7f03ea99c3fa679e1160487ad19131b174fbdbd5658f222e53ef']) namespacegcworker stdout | 2025-11-04 09:06:54,064 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,066 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:6c177c7cb8fd7f03ea99c3fa679e1160487ad19131b174fbdbd5658f222e53ef', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,067 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:54,067 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:6c177c7cb8fd7f03ea99c3fa679e1160487ad19131b174fbdbd5658f222e53ef namespacegcworker stdout | 2025-11-04 09:06:54,072 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:6c177c7cb8fd7f03ea99c3fa679e1160487ad19131b174fbdbd5658f222e53ef HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,072 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:6c177c7cb8fd7f03ea99c3fa679e1160487ad19131b174fbdbd5658f222e53ef from secscan service namespacegcworker stdout | 2025-11-04 09:06:54,073 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,074 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,075 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,076 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,077 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,078 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,079 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,080 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,081 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,082 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,083 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [16, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,084 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [16, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,085 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [16, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,086 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [16, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,086 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,087 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:54,087 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:54,087 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [16, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,088 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:4b2235713e64d414107b901ceb31c8f53c55b66346170aad3feed8a916a7a5b9']) namespacegcworker stdout | 2025-11-04 09:06:54,088 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,090 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:4b2235713e64d414107b901ceb31c8f53c55b66346170aad3feed8a916a7a5b9', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,091 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:54,091 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:4b2235713e64d414107b901ceb31c8f53c55b66346170aad3feed8a916a7a5b9 namespacegcworker stdout | 2025-11-04 09:06:54,103 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:4b2235713e64d414107b901ceb31c8f53c55b66346170aad3feed8a916a7a5b9 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,103 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:4b2235713e64d414107b901ceb31c8f53c55b66346170aad3feed8a916a7a5b9 from secscan service namespacegcworker stdout | 2025-11-04 09:06:54,104 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,105 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,105 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,107 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,108 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,109 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,110 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,111 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,112 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,113 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,114 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [17, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,114 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [17, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,115 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [17, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,115 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [17, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,116 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,116 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [3, 1]) namespacegcworker stdout | 2025-11-04 09:06:54,117 [75] [DEBUG] [data.model.quota] ineligible namespace 3 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:54,117 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [17, 7]) namespacegcworker stdout | 2025-11-04 09:06:54,117 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [7, 'sha256:b10cbe44bc0377ff7e0942c2e9ed9a655d050e577ac4bc8852b7868cc304f098']) namespacegcworker stdout | 2025-11-04 09:06:54,118 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,120 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:b10cbe44bc0377ff7e0942c2e9ed9a655d050e577ac4bc8852b7868cc304f098', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,120 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:54,120 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:b10cbe44bc0377ff7e0942c2e9ed9a655d050e577ac4bc8852b7868cc304f098 namespacegcworker stdout | 2025-11-04 09:06:54,125 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:b10cbe44bc0377ff7e0942c2e9ed9a655d050e577ac4bc8852b7868cc304f098 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,125 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:b10cbe44bc0377ff7e0942c2e9ed9a655d050e577ac4bc8852b7868cc304f098 from secscan service namespacegcworker stdout | 2025-11-04 09:06:54,126 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,126 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [1]) namespacegcworker stdout | 2025-11-04 09:06:54,129 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,129 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:54,131 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [3, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,132 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:54,134 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [4, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,135 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [4]) namespacegcworker stdout | 2025-11-04 09:06:54,136 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,137 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:54,138 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [6, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,139 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [6]) namespacegcworker stdout | 2025-11-04 09:06:54,140 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,141 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:54,143 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,143 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,145 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,146 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:54,147 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,148 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,149 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,150 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,152 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,152 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,154 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,155 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,156 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,157 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,159 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,159 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,161 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,161 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,163 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,163 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,165 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,166 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,167 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,168 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,169 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,170 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,171 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,172 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,174 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,174 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,176 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,176 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,178 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,179 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,181 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,181 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,183 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,184 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,186 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,186 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,188 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,189 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:54,191 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,191 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:54,194 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,194 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:54,197 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,198 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:54,200 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,201 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:54,202 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,203 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [33]) namespacegcworker stdout | 2025-11-04 09:06:54,205 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,206 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [34]) namespacegcworker stdout | 2025-11-04 09:06:54,208 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,208 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [35]) namespacegcworker stdout | 2025-11-04 09:06:54,210 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,211 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:54,213 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,214 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:54,216 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,216 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:54,218 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,218 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:54,220 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,221 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:54,223 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,224 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:54,226 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,226 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:54,229 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,229 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:54,231 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,231 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:54,233 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,234 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:54,235 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,236 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:54,238 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,239 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:54,241 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,241 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:54,243 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,243 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:54,245 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,245 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:54,247 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,248 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:54,250 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,250 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:54,252 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,252 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:54,254 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,254 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:54,257 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,258 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:54,259 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,260 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:54,261 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47} namespacegcworker stdout | 2025-11-04 09:06:54,261 [75] [DEBUG] [data.model.storage] Garbage collecting storage 8 namespacegcworker stdout | 2025-11-04 09:06:54,262 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,263 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,264 [75] [DEBUG] [data.model.storage] Garbage collecting storage 10 namespacegcworker stdout | 2025-11-04 09:06:54,264 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,265 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,265 [75] [DEBUG] [data.model.storage] Garbage collecting storage 11 namespacegcworker stdout | 2025-11-04 09:06:54,265 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,266 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,267 [75] [DEBUG] [data.model.storage] Garbage collecting storage 12 namespacegcworker stdout | 2025-11-04 09:06:54,267 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,268 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,269 [75] [DEBUG] [data.model.storage] Garbage collecting storage 13 namespacegcworker stdout | 2025-11-04 09:06:54,270 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,270 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,271 [75] [DEBUG] [data.model.storage] Garbage collecting storage 14 namespacegcworker stdout | 2025-11-04 09:06:54,271 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,272 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,272 [75] [DEBUG] [data.model.storage] Garbage collecting storage 15 namespacegcworker stdout | 2025-11-04 09:06:54,272 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,273 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,274 [75] [DEBUG] [data.model.storage] Garbage collecting storage 16 namespacegcworker stdout | 2025-11-04 09:06:54,274 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,275 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,276 [75] [DEBUG] [data.model.storage] Garbage collecting storage 17 namespacegcworker stdout | 2025-11-04 09:06:54,276 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,277 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,277 [75] [DEBUG] [data.model.storage] Garbage collecting storage 18 namespacegcworker stdout | 2025-11-04 09:06:54,277 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,279 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,280 [75] [DEBUG] [data.model.storage] Garbage collecting storage 19 namespacegcworker stdout | 2025-11-04 09:06:54,280 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,281 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,281 [75] [DEBUG] [data.model.storage] Garbage collecting storage 20 namespacegcworker stdout | 2025-11-04 09:06:54,281 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,282 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,283 [75] [DEBUG] [data.model.storage] Garbage collecting storage 21 namespacegcworker stdout | 2025-11-04 09:06:54,283 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,284 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,284 [75] [DEBUG] [data.model.storage] Garbage collecting storage 22 namespacegcworker stdout | 2025-11-04 09:06:54,285 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,286 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,286 [75] [DEBUG] [data.model.storage] Garbage collecting storage 23 namespacegcworker stdout | 2025-11-04 09:06:54,287 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,287 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,288 [75] [DEBUG] [data.model.storage] Garbage collecting storage 24 namespacegcworker stdout | 2025-11-04 09:06:54,288 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,289 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,289 [75] [DEBUG] [data.model.storage] Garbage collecting storage 25 namespacegcworker stdout | 2025-11-04 09:06:54,289 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,290 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,290 [75] [DEBUG] [data.model.storage] Garbage collecting storage 26 namespacegcworker stdout | 2025-11-04 09:06:54,291 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,292 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,292 [75] [DEBUG] [data.model.storage] Garbage collecting storage 27 namespacegcworker stdout | 2025-11-04 09:06:54,292 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,293 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,293 [75] [DEBUG] [data.model.storage] Garbage collecting storage 28 namespacegcworker stdout | 2025-11-04 09:06:54,294 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,295 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,295 [75] [DEBUG] [data.model.storage] Garbage collecting storage 29 namespacegcworker stdout | 2025-11-04 09:06:54,295 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,296 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,296 [75] [DEBUG] [data.model.storage] Garbage collecting storage 30 namespacegcworker stdout | 2025-11-04 09:06:54,296 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,297 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,297 [75] [DEBUG] [data.model.storage] Garbage collecting storage 31 namespacegcworker stdout | 2025-11-04 09:06:54,298 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,298 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,299 [75] [DEBUG] [data.model.storage] Garbage collecting storage 32 namespacegcworker stdout | 2025-11-04 09:06:54,299 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,300 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,300 [75] [DEBUG] [data.model.storage] Garbage collecting storage 33 namespacegcworker stdout | 2025-11-04 09:06:54,300 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,301 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,301 [75] [DEBUG] [data.model.storage] Garbage collecting storage 34 namespacegcworker stdout | 2025-11-04 09:06:54,301 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,302 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,302 [75] [DEBUG] [data.model.storage] Garbage collecting storage 35 namespacegcworker stdout | 2025-11-04 09:06:54,303 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,304 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,304 [75] [DEBUG] [data.model.storage] Garbage collecting storage 36 namespacegcworker stdout | 2025-11-04 09:06:54,305 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,305 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,306 [75] [DEBUG] [data.model.storage] Garbage collecting storage 37 namespacegcworker stdout | 2025-11-04 09:06:54,306 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,307 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,307 [75] [DEBUG] [data.model.storage] Garbage collecting storage 38 namespacegcworker stdout | 2025-11-04 09:06:54,307 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,308 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,308 [75] [DEBUG] [data.model.storage] Garbage collecting storage 39 namespacegcworker stdout | 2025-11-04 09:06:54,308 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,309 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,310 [75] [DEBUG] [data.model.storage] Garbage collecting storage 40 namespacegcworker stdout | 2025-11-04 09:06:54,310 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,311 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,311 [75] [DEBUG] [data.model.storage] Garbage collecting storage 41 namespacegcworker stdout | 2025-11-04 09:06:54,311 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,312 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,312 [75] [DEBUG] [data.model.storage] Garbage collecting storage 42 namespacegcworker stdout | 2025-11-04 09:06:54,313 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,313 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,314 [75] [DEBUG] [data.model.storage] Garbage collecting storage 43 namespacegcworker stdout | 2025-11-04 09:06:54,314 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,315 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,315 [75] [DEBUG] [data.model.storage] Garbage collecting storage 44 namespacegcworker stdout | 2025-11-04 09:06:54,315 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,316 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,316 [75] [DEBUG] [data.model.storage] Garbage collecting storage 45 namespacegcworker stdout | 2025-11-04 09:06:54,317 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,317 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,318 [75] [DEBUG] [data.model.storage] Garbage collecting storage 46 namespacegcworker stdout | 2025-11-04 09:06:54,318 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,319 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,319 [75] [DEBUG] [data.model.storage] Garbage collecting storage 47 namespacegcworker stdout | 2025-11-04 09:06:54,320 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,320 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,321 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47} namespacegcworker stdout | 2025-11-04 09:06:54,321 [75] [DEBUG] [data.model.storage] Garbage collecting storage 8 namespacegcworker stdout | 2025-11-04 09:06:54,321 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,322 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,322 [75] [DEBUG] [data.model.storage] Garbage collecting storage 10 namespacegcworker stdout | 2025-11-04 09:06:54,322 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,323 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,323 [75] [DEBUG] [data.model.storage] Garbage collecting storage 11 namespacegcworker stdout | 2025-11-04 09:06:54,324 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,324 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,325 [75] [DEBUG] [data.model.storage] Garbage collecting storage 12 namespacegcworker stdout | 2025-11-04 09:06:54,325 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,326 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,326 [75] [DEBUG] [data.model.storage] Garbage collecting storage 13 namespacegcworker stdout | 2025-11-04 09:06:54,326 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,327 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,327 [75] [DEBUG] [data.model.storage] Garbage collecting storage 14 namespacegcworker stdout | 2025-11-04 09:06:54,328 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,329 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,329 [75] [DEBUG] [data.model.storage] Garbage collecting storage 15 namespacegcworker stdout | 2025-11-04 09:06:54,330 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,330 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,331 [75] [DEBUG] [data.model.storage] Garbage collecting storage 16 namespacegcworker stdout | 2025-11-04 09:06:54,331 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,331 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,332 [75] [DEBUG] [data.model.storage] Garbage collecting storage 17 namespacegcworker stdout | 2025-11-04 09:06:54,333 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,333 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,334 [75] [DEBUG] [data.model.storage] Garbage collecting storage 18 namespacegcworker stdout | 2025-11-04 09:06:54,334 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,335 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,335 [75] [DEBUG] [data.model.storage] Garbage collecting storage 19 namespacegcworker stdout | 2025-11-04 09:06:54,336 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,336 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,337 [75] [DEBUG] [data.model.storage] Garbage collecting storage 20 namespacegcworker stdout | 2025-11-04 09:06:54,337 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,338 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,338 [75] [DEBUG] [data.model.storage] Garbage collecting storage 21 namespacegcworker stdout | 2025-11-04 09:06:54,338 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,339 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,339 [75] [DEBUG] [data.model.storage] Garbage collecting storage 22 namespacegcworker stdout | 2025-11-04 09:06:54,340 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,340 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,341 [75] [DEBUG] [data.model.storage] Garbage collecting storage 23 namespacegcworker stdout | 2025-11-04 09:06:54,341 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,342 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,342 [75] [DEBUG] [data.model.storage] Garbage collecting storage 24 namespacegcworker stdout | 2025-11-04 09:06:54,343 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,343 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,344 [75] [DEBUG] [data.model.storage] Garbage collecting storage 25 namespacegcworker stdout | 2025-11-04 09:06:54,344 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,345 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,345 [75] [DEBUG] [data.model.storage] Garbage collecting storage 26 namespacegcworker stdout | 2025-11-04 09:06:54,346 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,346 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,347 [75] [DEBUG] [data.model.storage] Garbage collecting storage 27 namespacegcworker stdout | 2025-11-04 09:06:54,347 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,348 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,348 [75] [DEBUG] [data.model.storage] Garbage collecting storage 28 namespacegcworker stdout | 2025-11-04 09:06:54,348 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,349 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,349 [75] [DEBUG] [data.model.storage] Garbage collecting storage 29 namespacegcworker stdout | 2025-11-04 09:06:54,350 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,350 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,351 [75] [DEBUG] [data.model.storage] Garbage collecting storage 30 namespacegcworker stdout | 2025-11-04 09:06:54,351 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,352 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,352 [75] [DEBUG] [data.model.storage] Garbage collecting storage 31 namespacegcworker stdout | 2025-11-04 09:06:54,352 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,353 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,354 [75] [DEBUG] [data.model.storage] Garbage collecting storage 32 namespacegcworker stdout | 2025-11-04 09:06:54,354 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,355 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,355 [75] [DEBUG] [data.model.storage] Garbage collecting storage 33 namespacegcworker stdout | 2025-11-04 09:06:54,355 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,356 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,357 [75] [DEBUG] [data.model.storage] Garbage collecting storage 34 namespacegcworker stdout | 2025-11-04 09:06:54,358 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,358 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,359 [75] [DEBUG] [data.model.storage] Garbage collecting storage 35 namespacegcworker stdout | 2025-11-04 09:06:54,359 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,360 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,360 [75] [DEBUG] [data.model.storage] Garbage collecting storage 36 namespacegcworker stdout | 2025-11-04 09:06:54,361 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,361 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,362 [75] [DEBUG] [data.model.storage] Garbage collecting storage 37 namespacegcworker stdout | 2025-11-04 09:06:54,362 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,363 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,363 [75] [DEBUG] [data.model.storage] Garbage collecting storage 38 namespacegcworker stdout | 2025-11-04 09:06:54,363 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,364 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,364 [75] [DEBUG] [data.model.storage] Garbage collecting storage 39 namespacegcworker stdout | 2025-11-04 09:06:54,365 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,365 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,366 [75] [DEBUG] [data.model.storage] Garbage collecting storage 40 namespacegcworker stdout | 2025-11-04 09:06:54,366 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,367 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,367 [75] [DEBUG] [data.model.storage] Garbage collecting storage 41 namespacegcworker stdout | 2025-11-04 09:06:54,368 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,368 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,369 [75] [DEBUG] [data.model.storage] Garbage collecting storage 42 namespacegcworker stdout | 2025-11-04 09:06:54,369 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,370 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,370 [75] [DEBUG] [data.model.storage] Garbage collecting storage 43 namespacegcworker stdout | 2025-11-04 09:06:54,371 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,371 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,372 [75] [DEBUG] [data.model.storage] Garbage collecting storage 44 namespacegcworker stdout | 2025-11-04 09:06:54,372 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,373 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,373 [75] [DEBUG] [data.model.storage] Garbage collecting storage 45 namespacegcworker stdout | 2025-11-04 09:06:54,373 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,374 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,374 [75] [DEBUG] [data.model.storage] Garbage collecting storage 46 namespacegcworker stdout | 2025-11-04 09:06:54,375 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,376 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,377 [75] [DEBUG] [data.model.storage] Garbage collecting storage 47 namespacegcworker stdout | 2025-11-04 09:06:54,377 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,377 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,378 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:54,379 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:54,379 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:54,380 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:54,381 [75] [DEBUG] [data.model.gc] Found 10 uploaded blobs to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,381 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 8 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,381 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,383 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 9 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,383 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [9]) namespacegcworker stdout | 2025-11-04 09:06:54,385 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 10 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,385 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,386 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 11 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,386 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,388 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 12 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,388 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,389 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 13 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,389 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,391 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 14 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,391 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,392 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 15 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,392 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,394 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 16 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,394 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,395 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 17 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,395 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,397 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {8, 10, 11, 12, 13, 14, 15, 16, 17, 18} namespacegcworker stdout | 2025-11-04 09:06:54,397 [75] [DEBUG] [data.model.storage] Garbage collecting storage 8 namespacegcworker stdout | 2025-11-04 09:06:54,397 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,398 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,398 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,399 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,400 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,401 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:54,402 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236']) namespacegcworker stdout | 2025-11-04 09:06:54,402 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236']) namespacegcworker stdout | 2025-11-04 09:06:54,403 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "imagestoragelocation" AS "t1"', []) namespacegcworker stdout | 2025-11-04 09:06:54,405 [75] [DEBUG] [data.model.storage] Garbage collecting storage 10 namespacegcworker stdout | 2025-11-04 09:06:54,405 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,406 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,407 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,407 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,407 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,408 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [10]) namespacegcworker stdout | 2025-11-04 09:06:54,408 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61']) namespacegcworker stdout | 2025-11-04 09:06:54,409 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61']) namespacegcworker stdout | 2025-11-04 09:06:54,410 [75] [DEBUG] [data.model.storage] Garbage collecting storage 11 namespacegcworker stdout | 2025-11-04 09:06:54,410 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,411 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [11, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,412 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,412 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,413 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,413 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [11]) namespacegcworker stdout | 2025-11-04 09:06:54,414 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944']) namespacegcworker stdout | 2025-11-04 09:06:54,414 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944']) namespacegcworker stdout | 2025-11-04 09:06:54,416 [75] [DEBUG] [data.model.storage] Garbage collecting storage 12 namespacegcworker stdout | 2025-11-04 09:06:54,416 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,416 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [12, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,417 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,418 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,419 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,419 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [12]) namespacegcworker stdout | 2025-11-04 09:06:54,420 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a']) namespacegcworker stdout | 2025-11-04 09:06:54,420 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a']) namespacegcworker stdout | 2025-11-04 09:06:54,422 [75] [DEBUG] [data.model.storage] Garbage collecting storage 13 namespacegcworker stdout | 2025-11-04 09:06:54,423 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,423 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [13, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,424 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,425 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,425 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,426 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [13]) namespacegcworker stdout | 2025-11-04 09:06:54,427 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36']) namespacegcworker stdout | 2025-11-04 09:06:54,427 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36']) namespacegcworker stdout | 2025-11-04 09:06:54,429 [75] [DEBUG] [data.model.storage] Garbage collecting storage 14 namespacegcworker stdout | 2025-11-04 09:06:54,429 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,430 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [14, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,431 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,431 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,432 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,432 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [14]) namespacegcworker stdout | 2025-11-04 09:06:54,433 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c']) namespacegcworker stdout | 2025-11-04 09:06:54,434 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c']) namespacegcworker stdout | 2025-11-04 09:06:54,435 [75] [DEBUG] [data.model.storage] Garbage collecting storage 15 namespacegcworker stdout | 2025-11-04 09:06:54,435 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,436 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [15, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,437 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,437 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,438 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,438 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [15]) namespacegcworker stdout | 2025-11-04 09:06:54,439 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5']) namespacegcworker stdout | 2025-11-04 09:06:54,439 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5']) namespacegcworker stdout | 2025-11-04 09:06:54,441 [75] [DEBUG] [data.model.storage] Garbage collecting storage 16 namespacegcworker stdout | 2025-11-04 09:06:54,441 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,442 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [16, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,443 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,443 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,444 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,445 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [16]) namespacegcworker stdout | 2025-11-04 09:06:54,445 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735']) namespacegcworker stdout | 2025-11-04 09:06:54,446 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735']) namespacegcworker stdout | 2025-11-04 09:06:54,447 [75] [DEBUG] [data.model.storage] Garbage collecting storage 17 namespacegcworker stdout | 2025-11-04 09:06:54,448 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,448 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [17, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,449 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,449 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,450 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,450 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [17]) namespacegcworker stdout | 2025-11-04 09:06:54,451 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9']) namespacegcworker stdout | 2025-11-04 09:06:54,451 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9']) namespacegcworker stdout | 2025-11-04 09:06:54,453 [75] [DEBUG] [data.model.storage] Garbage collecting storage 18 namespacegcworker stdout | 2025-11-04 09:06:54,453 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,454 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [18, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,454 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,455 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,455 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,455 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,456 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9']) namespacegcworker stdout | 2025-11-04 09:06:54,456 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9']) namespacegcworker stdout | 2025-11-04 09:06:54,458 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,458 [75] [DEBUG] [data.model.storage] Removing sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,460 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/endpoints.json namespacegcworker stdout | 2025-11-04 09:06:54,476 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/sdk-default-configuration.json namespacegcworker stdout | 2025-11-04 09:06:54,476 [75] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,492 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/service-2.json namespacegcworker stdout | 2025-11-04 09:06:54,512 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/endpoint-rule-set-1.json namespacegcworker stdout | 2025-11-04 09:06:54,515 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/partitions.json namespacegcworker stdout | 2025-11-04 09:06:54,517 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,517 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7fbe73e4ab60> namespacegcworker stdout | 2025-11-04 09:06:54,530 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,531 [75] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) namespacegcworker stdout | 2025-11-04 09:06:54,533 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/_retry.json namespacegcworker stdout | 2025-11-04 09:06:54,534 [75] [DEBUG] [botocore.client] Registering retry handlers for service: s3 namespacegcworker stdout | 2025-11-04 09:06:54,534 [75] [DEBUG] [botocore.utils] Registering S3 region redirector handler namespacegcworker stdout | 2025-11-04 09:06:54,549 [75] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/boto3/data/s3/2006-03-01/resources-1.json namespacegcworker stdout | 2025-11-04 09:06:54,550 [75] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,551 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,551 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7fbe73e4ab60> namespacegcworker stdout | 2025-11-04 09:06:54,551 [75] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,554 [75] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) namespacegcworker stdout | 2025-11-04 09:06:54,556 [75] [DEBUG] [botocore.client] Registering retry handlers for service: s3 namespacegcworker stdout | 2025-11-04 09:06:54,638 [75] [DEBUG] [botocore.utils] Registering S3 region redirector handler namespacegcworker stdout | 2025-11-04 09:06:54,639 [75] [DEBUG] [boto3.resources.factory] Loading s3:s3 namespacegcworker stdout | 2025-11-04 09:06:54,639 [75] [DEBUG] [boto3.resources.factory] Loading s3:Bucket namespacegcworker stdout | 2025-11-04 09:06:54,639 [75] [DEBUG] [boto3.resources.model] Renaming Bucket attribute name namespacegcworker stdout | 2025-11-04 09:06:54,640 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Bucket: calling handler ._handler at 0x7fbe73e4ae80> namespacegcworker stdout | 2025-11-04 09:06:54,640 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,640 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,640 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,641 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,641 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,641 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,641 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,642 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadBucket) with params: {'url_path': '', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b158578ea44b8c8a543d5e36e6e02b8e19cf274f832b146694c4d72ff119a4d2 namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 4786a03fd7e528554ba8a9adf56f72c29e378e8116d691bd94b7bc6f8bdd5738 namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,643 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,644 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,644 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4786a03fd7e528554ba8a9adf56f72c29e378e8116d691bd94b7bc6f8bdd5738', 'amz-sdk-invocation-id': b'4eec346d-10dd-4e07-a37d-47000ff146f5', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,644 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,644 [75] [DEBUG] [urllib3.connectionpool] Starting new HTTPS connection (1): s3.openshift-storage.svc.cluster.local:443 namespacegcworker stdout | 2025-11-04 09:06:54,689 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,689 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpav-e2gc59-17gr', 'x-amz-id-2': 'mhkchpav-e2gc59-17gr', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,689 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,689 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,690 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,690 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,690 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadBucket: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,690 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,691 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,691 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944'} namespacegcworker stdout | 2025-11-04 09:06:54,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,692 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,693 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 948dba1ec99559fc957b697a0d4651ef94cf0e8feb7c42206068daef563b0fda namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | c6d306bc03cdc3b886e1aa0196fca86a250477018104ba0b9c5f49d7f1a335d7 namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c6d306bc03cdc3b886e1aa0196fca86a250477018104ba0b9c5f49d7f1a335d7', 'amz-sdk-invocation-id': b'ad34898f-598d-4c88-afef-3e6492aeab4a', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,694 [75] [DEBUG] [urllib3.connectionpool] Starting new HTTPS connection (1): s3.openshift-storage.svc.cluster.local:443 namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpc3-esn61l-mbd', 'x-amz-id-2': 'mhkchpc3-esn61l-mbd', 'ETag': '"9272f5e1149a6205e203feea94919550-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:32 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5682', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,727 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpc3-esn61l-mbd', 'HostId': 'mhkchpc3-esn61l-mbd', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpc3-esn61l-mbd', 'x-amz-id-2': 'mhkchpc3-esn61l-mbd', 'etag': '"9272f5e1149a6205e203feea94919550-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:32 GMT', 'content-type': 'application/octet-stream', 'content-length': '5682', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 32, tzinfo=tzutc()), 'ContentLength': 5682, 'ETag': '"9272f5e1149a6205e203feea94919550-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944'} namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,728 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | a188daa91fdd00cee32a39a393e15a0d85cf0c04f2194fc7ef812baad6dcc2ba namespacegcworker stdout | 2025-11-04 09:06:54,729 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | fb69333de9e5b9326795ae6f16899dd02fcf1875ce13624c3ed8b6640d881c52 namespacegcworker stdout | 2025-11-04 09:06:54,730 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,730 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,730 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,730 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=fb69333de9e5b9326795ae6f16899dd02fcf1875ce13624c3ed8b6640d881c52', 'amz-sdk-invocation-id': b'75a2af9b-6b32-4183-82b0-54915c432984', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,730 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,740 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/07/0775bf52e81a3cf747ce4b00c80d91adc63a6a121d89b946e8b5a0c9ba509944 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcb-ex7cf9-p9d', 'x-amz-id-2': 'mhkchpcb-ex7cf9-p9d', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpcb-ex7cf9-p9d', 'HostId': 'mhkchpcb-ex7cf9-p9d', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcb-ex7cf9-p9d', 'x-amz-id-2': 'mhkchpcb-ex7cf9-p9d', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,741 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,742 [75] [DEBUG] [data.model.storage] Removing sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,742 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9'} namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,743 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,744 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 75b632440b27fea32ef1ad28e867de0f96da9bc83cde4981f866343723ef5c42 namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | cda51d4e4662c8efd94aafc7ec118413fbb64120fcfdb3842a0dfcf4a66eebec namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=cda51d4e4662c8efd94aafc7ec118413fbb64120fcfdb3842a0dfcf4a66eebec', 'amz-sdk-invocation-id': b'f0ddba0f-83b7-4e8b-b5fd-c98c141c2bd7', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,745 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcq-f65tup-14rc', 'x-amz-id-2': 'mhkchpcq-f65tup-14rc', 'ETag': '"933e733f8b98d678802844b56ef9efd0-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:41 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3219181', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpcq-f65tup-14rc', 'HostId': 'mhkchpcq-f65tup-14rc', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcq-f65tup-14rc', 'x-amz-id-2': 'mhkchpcq-f65tup-14rc', 'etag': '"933e733f8b98d678802844b56ef9efd0-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:41 GMT', 'content-type': 'application/octet-stream', 'content-length': '3219181', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 41, tzinfo=tzutc()), 'ContentLength': 3219181, 'ETag': '"933e733f8b98d678802844b56ef9efd0-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9'} namespacegcworker stdout | 2025-11-04 09:06:54,749 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 42e92964a60ff44e51ef22dbf3cc25c8b86ddafc2be4603a9102502943f44c8a namespacegcworker stdout | 2025-11-04 09:06:54,750 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 550be793349e34434bd416a02dadafd82b525f8cf81c52bb3fa5fd5221afd34d namespacegcworker stdout | 2025-11-04 09:06:54,751 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,751 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,751 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,751 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=550be793349e34434bd416a02dadafd82b525f8cf81c52bb3fa5fd5221afd34d', 'amz-sdk-invocation-id': b'ff85557f-1c8a-46f3-84f6-bb9402c19b30', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,751 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,757 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/22e4d17029cf647ff505d5389be90006efc5ed4178aed9a6d798a2bf7a675fc9 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,757 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcw-f9nr5m-jeo', 'x-amz-id-2': 'mhkchpcw-f9nr5m-jeo', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpcw-f9nr5m-jeo', 'HostId': 'mhkchpcw-f9nr5m-jeo', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpcw-f9nr5m-jeo', 'x-amz-id-2': 'mhkchpcw-f9nr5m-jeo', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,758 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,759 [75] [DEBUG] [data.model.storage] Removing sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,759 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61'} namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,760 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 3fae4ab8e7e466456cabf561c83a78482ea37556417253a99067fa8896c850c6 namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3bfeec2eea29d52006ad21f66ce93f8a6e7ee9bd9177f453ea70335d0f4a8680 namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3bfeec2eea29d52006ad21f66ce93f8a6e7ee9bd9177f453ea70335d0f4a8680', 'amz-sdk-invocation-id': b'78fba2b0-6de0-494b-877a-b6efe1785483', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,761 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpd6-ffy4ne-84p', 'x-amz-id-2': 'mhkchpd6-ffy4ne-84p', 'ETag': '"c953f11b655c667e87bf2e6d854fb992-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:30 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '581', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,766 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpd6-ffy4ne-84p', 'HostId': 'mhkchpd6-ffy4ne-84p', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpd6-ffy4ne-84p', 'x-amz-id-2': 'mhkchpd6-ffy4ne-84p', 'etag': '"c953f11b655c667e87bf2e6d854fb992-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:30 GMT', 'content-type': 'application/octet-stream', 'content-length': '581', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 30, tzinfo=tzutc()), 'ContentLength': 581, 'ETag': '"c953f11b655c667e87bf2e6d854fb992-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61'} namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,767 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 8a69ad4098b2b1c9c3791daa5465f1af05c9383d90fd70f8ded3c9e773a0350b namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3af03c4371279382c18171ba45a9ed4e5291f1017e163b651902cad8c6deac68 namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3af03c4371279382c18171ba45a9ed4e5291f1017e163b651902cad8c6deac68', 'amz-sdk-invocation-id': b'6826bfaa-7a84-4d4c-9273-89d2c62d714a', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,768 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ce/cea2ff433c610f5363017404ce989632e12b953114fefc6f597a58e813c15d61 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdd-fjsdf0-ytw', 'x-amz-id-2': 'mhkchpdd-fjsdf0-ytw', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,775 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpdd-fjsdf0-ytw', 'HostId': 'mhkchpdd-fjsdf0-ytw', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdd-fjsdf0-ytw', 'x-amz-id-2': 'mhkchpdd-fjsdf0-ytw', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,776 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,776 [75] [DEBUG] [data.model.storage] Removing sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c from local_us namespacegcworker stdout | 2025-11-04 09:06:54,776 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c'} namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,777 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | db76530ada196086f7dd07187dc232bac175876cea37f21673ee41cdfb7a48b7 namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 761c5072bf0d7b645f85681168720d683112be041721f1208bc420ea93b04464 namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=761c5072bf0d7b645f85681168720d683112be041721f1208bc420ea93b04464', 'amz-sdk-invocation-id': b'bb068c64-04dc-4878-8014-be24c3f41768', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,778 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,782 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdn-fpzdro-14bh', 'x-amz-id-2': 'mhkchpdn-fpzdro-14bh', 'ETag': '"15c25308dfd7d923db6b0c81593d4b12-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:36 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3500929', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpdn-fpzdro-14bh', 'HostId': 'mhkchpdn-fpzdro-14bh', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdn-fpzdro-14bh', 'x-amz-id-2': 'mhkchpdn-fpzdro-14bh', 'etag': '"15c25308dfd7d923db6b0c81593d4b12-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:36 GMT', 'content-type': 'application/octet-stream', 'content-length': '3500929', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 36, tzinfo=tzutc()), 'ContentLength': 3500929, 'ETag': '"15c25308dfd7d923db6b0c81593d4b12-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c'} namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,783 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b85deb2bcbc101df672d86420f803f0f0c12b94806d64b2e5de4431a4815acc1 namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 95609d65d91caeb0187263fbc609b98a13c45ce3afac59783e8fb29d4661fc0b namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,784 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=95609d65d91caeb0187263fbc609b98a13c45ce3afac59783e8fb29d4661fc0b', 'amz-sdk-invocation-id': b'9d623ebf-55f5-416e-b825-22454060bbdf', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,785 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5d/5ddfb4a71b19e6dcd52b9c46193b6249cf9b39300f0f664f0d682463a4d48e6c HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdt-ftp3na-18k9', 'x-amz-id-2': 'mhkchpdt-ftp3na-18k9', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,792 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,793 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpdt-ftp3na-18k9', 'HostId': 'mhkchpdt-ftp3na-18k9', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpdt-ftp3na-18k9', 'x-amz-id-2': 'mhkchpdt-ftp3na-18k9', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,793 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,794 [75] [DEBUG] [data.model.storage] Removing sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a from local_us namespacegcworker stdout | 2025-11-04 09:06:54,794 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,794 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,794 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a'} namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,795 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e39740649eb75c0a2a98d3e2655231b653d8c6bb48adb2075944b57d43d8a824 namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 61b7cc62ea979c1624a29c0ef6c9985a80e1e1f25365f7159b01a81fe21ad8de namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=61b7cc62ea979c1624a29c0ef6c9985a80e1e1f25365f7159b01a81fe21ad8de', 'amz-sdk-invocation-id': b'8bdaaf85-f325-45d2-b676-085b8aaee9a4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,796 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpe5-g0g7dc-ze0', 'x-amz-id-2': 'mhkchpe5-g0g7dc-ze0', 'ETag': '"cfca356d6af89691a600aef06e451bc6-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:32 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80956', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,800 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpe5-g0g7dc-ze0', 'HostId': 'mhkchpe5-g0g7dc-ze0', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpe5-g0g7dc-ze0', 'x-amz-id-2': 'mhkchpe5-g0g7dc-ze0', 'etag': '"cfca356d6af89691a600aef06e451bc6-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:32 GMT', 'content-type': 'application/octet-stream', 'content-length': '80956', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 32, tzinfo=tzutc()), 'ContentLength': 80956, 'ETag': '"cfca356d6af89691a600aef06e451bc6-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a'} namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,801 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | fd8cb2846e8b6c994c56eab442f5cc11e688fbd43c617ee00d3c3fcdb83510ed namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 8e2aea650e879234bd60165c0f7adc145e97082ec3f1c108dd2e403aeb523110 namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8e2aea650e879234bd60165c0f7adc145e97082ec3f1c108dd2e403aeb523110', 'amz-sdk-invocation-id': b'a42c6b43-9762-4654-b3ce-30aca96b2bdc', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,802 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5f/5ff8d76a871d143e3d5bf83dd9a47bdd130b4f9c66d34531b064f0395b77ea6a HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpeb-g41tw2-101v', 'x-amz-id-2': 'mhkchpeb-g41tw2-101v', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpeb-g41tw2-101v', 'HostId': 'mhkchpeb-g41tw2-101v', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpeb-g41tw2-101v', 'x-amz-id-2': 'mhkchpeb-g41tw2-101v', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,808 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,809 [75] [DEBUG] [data.model.storage] Removing sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,809 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36'} namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,810 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2cb5ff3de26ea286082dc66c0f2554cebfdc533a17221ce49ebaecf9f3e60202 namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | fcd0f35b350cfac7ca865e29579af8185d2067711ff2d274942041a11cf87020 namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=fcd0f35b350cfac7ca865e29579af8185d2067711ff2d274942041a11cf87020', 'amz-sdk-invocation-id': b'744f4f08-864d-4c28-b763-24b6ebb168b4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,811 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,814 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,814 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpek-g9gwe6-1ail', 'x-amz-id-2': 'mhkchpek-g9gwe6-1ail', 'ETag': '"c170324d07169ded9b90cce181613f90-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:34 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,814 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpek-g9gwe6-1ail', 'HostId': 'mhkchpek-g9gwe6-1ail', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpek-g9gwe6-1ail', 'x-amz-id-2': 'mhkchpek-g9gwe6-1ail', 'etag': '"c170324d07169ded9b90cce181613f90-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:34 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 34, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"c170324d07169ded9b90cce181613f90-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36'} namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,815 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 5fb6d4789e8efb13dc1a1b9350e2ecd67a627d7f6deda0704f0c68d2ae9b436e namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 10d0657526ff42ac79aafe8e9d0d5033903172dc96955689e6dc37cbd5676703 namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=10d0657526ff42ac79aafe8e9d0d5033903172dc96955689e6dc37cbd5676703', 'amz-sdk-invocation-id': b'10729c26-67c3-445c-87db-f2ca2e7bd666', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,816 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/7a/7a444bc9f0713d2f048cd64e3458f7ef8d29683814fcb08a3e5c8fc3c4b41d36 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpep-gcq5e7-zyt', 'x-amz-id-2': 'mhkchpep-gcq5e7-zyt', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpep-gcq5e7-zyt', 'HostId': 'mhkchpep-gcq5e7-zyt', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpep-gcq5e7-zyt', 'x-amz-id-2': 'mhkchpep-gcq5e7-zyt', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,823 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,824 [75] [DEBUG] [data.model.storage] Removing sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,824 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9'} namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,825 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2de141f42ba4f31fb5bdb0df2266c5ab662b60232d324a52dc692880b761eb2e namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | b474db42e582a5db5e5033a3d3a464312b686b84c3accae8539e22e2909c205e namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,826 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=b474db42e582a5db5e5033a3d3a464312b686b84c3accae8539e22e2909c205e', 'amz-sdk-invocation-id': b'ac361f51-d67e-460f-835f-c03b216920e4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,827 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpf0-girpw6-17lq', 'x-amz-id-2': 'mhkchpf0-girpw6-17lq', 'ETag': '"a9818db4e7d1861b8e5bdda91b9febaf-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:39 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '167', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,830 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpf0-girpw6-17lq', 'HostId': 'mhkchpf0-girpw6-17lq', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpf0-girpw6-17lq', 'x-amz-id-2': 'mhkchpf0-girpw6-17lq', 'etag': '"a9818db4e7d1861b8e5bdda91b9febaf-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:39 GMT', 'content-type': 'application/octet-stream', 'content-length': '167', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 39, tzinfo=tzutc()), 'ContentLength': 167, 'ETag': '"a9818db4e7d1861b8e5bdda91b9febaf-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9'} namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,831 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | ace341a6fed7bdbbb19d2e059411062915a2441118a5dab11d70fd49414d1e94 namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | bd1b49b4863375f59c3a8339a11c0a8d5e2dcc8779ea7d7cfed3fa534a7c996d namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=bd1b49b4863375f59c3a8339a11c0a8d5e2dcc8779ea7d7cfed3fa534a7c996d', 'amz-sdk-invocation-id': b'fcc0b54c-a9fe-4920-8f24-bb0a3c722938', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,832 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/54/54a440516c40bffd899eafa298d86b1b7319b62e4f732d8acb1e7c1b3e61a6f9 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpf5-2k6aa-1d5j', 'x-amz-id-2': 'mhkchpf5-2k6aa-1d5j', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,838 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,839 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpf5-2k6aa-1d5j', 'HostId': 'mhkchpf5-2k6aa-1d5j', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpf5-2k6aa-1d5j', 'x-amz-id-2': 'mhkchpf5-2k6aa-1d5j', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,839 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,839 [75] [DEBUG] [data.model.storage] Removing sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,839 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236'} namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,840 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | c7eef2afd39d7a1b0a1cbb3c15b39aebd65b349fb39781735538ee60655a36fe namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 097e39750a211e0f6cf2c01c433f900dcba08b60af61c07a9fa080e2d87862ea namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,841 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=097e39750a211e0f6cf2c01c433f900dcba08b60af61c07a9fa080e2d87862ea', 'amz-sdk-invocation-id': b'c5728f6e-9ba7-4f8a-952e-cc34b6c57ea0', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,842 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,845 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,845 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfe-88pt1-trl', 'x-amz-id-2': 'mhkchpfe-88pt1-trl', 'ETag': '"23084520a312a049f43be174125adc34-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:29 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3796846', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,845 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpfe-88pt1-trl', 'HostId': 'mhkchpfe-88pt1-trl', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfe-88pt1-trl', 'x-amz-id-2': 'mhkchpfe-88pt1-trl', 'etag': '"23084520a312a049f43be174125adc34-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:29 GMT', 'content-type': 'application/octet-stream', 'content-length': '3796846', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 29, tzinfo=tzutc()), 'ContentLength': 3796846, 'ETag': '"23084520a312a049f43be174125adc34-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236'} namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,846 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1fe5adcb70f08bead7afe623e2b0592c98623a440728c0eaa17a6b646ac644a8 namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 2ba0a5e5871cf50a01923e012e4eb4b92a181edf653cbcb4fc86bdc094d94b75 namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2ba0a5e5871cf50a01923e012e4eb4b92a181edf653cbcb4fc86bdc094d94b75', 'amz-sdk-invocation-id': b'512a1b81-31ba-46d7-8def-ade8ef3e6ec9', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,847 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fe/fe07684b16b82247c3539ed86a65ff37a76138ec25d380bd80c869a1a4c73236 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfk-blxgr-x5w', 'x-amz-id-2': 'mhkchpfk-blxgr-x5w', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpfk-blxgr-x5w', 'HostId': 'mhkchpfk-blxgr-x5w', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfk-blxgr-x5w', 'x-amz-id-2': 'mhkchpfk-blxgr-x5w', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,854 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,855 [75] [DEBUG] [data.model.storage] Removing sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,855 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,855 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735'} namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,856 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e137f491e79d9e98bc9d6c321c84b60e872114e03f8e908ef167045306e441b6 namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 02a4dc18b43f46688eea35fd1f5504167e7cefc2bece92cac4c733385004ecb5 namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=02a4dc18b43f46688eea35fd1f5504167e7cefc2bece92cac4c733385004ecb5', 'amz-sdk-invocation-id': b'2e53b9e7-bef2-42e3-b525-437dc3c77c29', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,857 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,860 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,860 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfu-hgmsm-e1', 'x-amz-id-2': 'mhkchpfu-hgmsm-e1', 'ETag': '"01968e4d19351d69b46abb0f49662431-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:38 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5504', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,860 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpfu-hgmsm-e1', 'HostId': 'mhkchpfu-hgmsm-e1', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpfu-hgmsm-e1', 'x-amz-id-2': 'mhkchpfu-hgmsm-e1', 'etag': '"01968e4d19351d69b46abb0f49662431-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:38 GMT', 'content-type': 'application/octet-stream', 'content-length': '5504', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 38, tzinfo=tzutc()), 'ContentLength': 5504, 'ETag': '"01968e4d19351d69b46abb0f49662431-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735'} namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,861 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6b75de31503142cd22a82fe45680c8f337a143f5317950b9faa154020846ed75 namespacegcworker stdout | 2025-11-04 09:06:54,862 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | a40230e7d9053020e5b648ce945297e9a247ea3d8a0d18725bc0991f367421f6 namespacegcworker stdout | 2025-11-04 09:06:54,863 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,863 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,863 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,863 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a40230e7d9053020e5b648ce945297e9a247ea3d8a0d18725bc0991f367421f6', 'amz-sdk-invocation-id': b'185668af-44c4-4d33-9a23-9e072239ee4f', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,863 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/61/611014955e734ee4b6df2d70e50ea5ef979fd32d6da80bd4f1c29a42a39dd735 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpg0-l62vk-fmf', 'x-amz-id-2': 'mhkchpg0-l62vk-fmf', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,869 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpg0-l62vk-fmf', 'HostId': 'mhkchpg0-l62vk-fmf', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpg0-l62vk-fmf', 'x-amz-id-2': 'mhkchpg0-l62vk-fmf', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,870 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,870 [75] [DEBUG] [data.model.storage] Removing sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,870 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5'} namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,871 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,872 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 379442ccdee6870e5048b168b430a4d70a91cf4e54050aab1ac1f117840f1396 namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 93d298b84bf8c7c8a71fd5f5c47392d4d7a41c4b327b971b65f070ec335b9bd9 namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=93d298b84bf8c7c8a71fd5f5c47392d4d7a41c4b327b971b65f070ec335b9bd9', 'amz-sdk-invocation-id': b'2b3cc82b-52f3-4118-8b60-c27a821b083e', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,873 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,876 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,876 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpga-qxi1b-ftu', 'x-amz-id-2': 'mhkchpga-qxi1b-ftu', 'ETag': '"fbce6db1c08805ef6b3a57204c111a92-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:36 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '593', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,876 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpga-qxi1b-ftu', 'HostId': 'mhkchpga-qxi1b-ftu', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpga-qxi1b-ftu', 'x-amz-id-2': 'mhkchpga-qxi1b-ftu', 'etag': '"fbce6db1c08805ef6b3a57204c111a92-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:36 GMT', 'content-type': 'application/octet-stream', 'content-length': '593', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 36, tzinfo=tzutc()), 'ContentLength': 593, 'ETag': '"fbce6db1c08805ef6b3a57204c111a92-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5'} namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,877 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 102875804db4a240dae0172dcb61b3b1f2aba6bda6c9fd17c5d8a216dd88d88d namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | cad15e511f9356b2c04cca13579bb268ffe6eaf4ca6265655b1f2d82c136d15e namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,878 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,879 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=cad15e511f9356b2c04cca13579bb268ffe6eaf4ca6265655b1f2d82c136d15e', 'amz-sdk-invocation-id': b'52110e9d-aaab-4cb4-bbc1-54d71333ee00', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,879 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/22/2283721971a18c443314dca49231fb058e4956cc1558113bbc3fc407ffa824b5 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpgg-ucj8a-kwf', 'x-amz-id-2': 'mhkchpgg-ucj8a-kwf', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpgg-ucj8a-kwf', 'HostId': 'mhkchpgg-ucj8a-kwf', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpgg-ucj8a-kwf', 'x-amz-id-2': 'mhkchpgg-ucj8a-kwf', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,885 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:54,886 [75] [DEBUG] [data.model.gc] Found 10 uploaded blobs to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,886 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 18 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,886 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [18]) namespacegcworker stdout | 2025-11-04 09:06:54,888 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 19 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,888 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,890 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 20 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,890 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,891 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 21 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,891 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,893 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 22 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,893 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,894 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 23 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,894 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,896 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 24 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,896 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,897 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 25 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,897 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,899 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 26 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,899 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,900 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 27 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:54,900 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,902 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {19, 20, 21, 22, 23, 24, 25, 26, 27, 28} namespacegcworker stdout | 2025-11-04 09:06:54,902 [75] [DEBUG] [data.model.storage] Garbage collecting storage 19 namespacegcworker stdout | 2025-11-04 09:06:54,902 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,903 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,904 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,905 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,906 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,906 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:54,907 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577']) namespacegcworker stdout | 2025-11-04 09:06:54,907 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577']) namespacegcworker stdout | 2025-11-04 09:06:54,908 [75] [DEBUG] [data.model.storage] Garbage collecting storage 20 namespacegcworker stdout | 2025-11-04 09:06:54,909 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,909 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,910 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,911 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,912 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,912 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:54,913 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba']) namespacegcworker stdout | 2025-11-04 09:06:54,914 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba']) namespacegcworker stdout | 2025-11-04 09:06:54,915 [75] [DEBUG] [data.model.storage] Garbage collecting storage 21 namespacegcworker stdout | 2025-11-04 09:06:54,915 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,916 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,917 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,918 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,918 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,919 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:54,919 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1']) namespacegcworker stdout | 2025-11-04 09:06:54,919 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1']) namespacegcworker stdout | 2025-11-04 09:06:54,921 [75] [DEBUG] [data.model.storage] Garbage collecting storage 22 namespacegcworker stdout | 2025-11-04 09:06:54,921 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,922 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,923 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,924 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,924 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,924 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:54,925 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07']) namespacegcworker stdout | 2025-11-04 09:06:54,925 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07']) namespacegcworker stdout | 2025-11-04 09:06:54,927 [75] [DEBUG] [data.model.storage] Garbage collecting storage 23 namespacegcworker stdout | 2025-11-04 09:06:54,927 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,928 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,929 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,930 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,931 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,931 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:54,932 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7']) namespacegcworker stdout | 2025-11-04 09:06:54,932 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7']) namespacegcworker stdout | 2025-11-04 09:06:54,934 [75] [DEBUG] [data.model.storage] Garbage collecting storage 24 namespacegcworker stdout | 2025-11-04 09:06:54,934 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,935 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,935 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,936 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,937 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,937 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:54,937 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a']) namespacegcworker stdout | 2025-11-04 09:06:54,938 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a']) namespacegcworker stdout | 2025-11-04 09:06:54,939 [75] [DEBUG] [data.model.storage] Garbage collecting storage 25 namespacegcworker stdout | 2025-11-04 09:06:54,940 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,941 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,942 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,942 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,943 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,943 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:54,944 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483']) namespacegcworker stdout | 2025-11-04 09:06:54,944 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483']) namespacegcworker stdout | 2025-11-04 09:06:54,946 [75] [DEBUG] [data.model.storage] Garbage collecting storage 26 namespacegcworker stdout | 2025-11-04 09:06:54,946 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,947 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,948 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,948 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,949 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,949 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:54,950 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4']) namespacegcworker stdout | 2025-11-04 09:06:54,950 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4']) namespacegcworker stdout | 2025-11-04 09:06:54,951 [75] [DEBUG] [data.model.storage] Garbage collecting storage 27 namespacegcworker stdout | 2025-11-04 09:06:54,952 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,952 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,953 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,954 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,954 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,955 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:54,956 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283']) namespacegcworker stdout | 2025-11-04 09:06:54,956 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283']) namespacegcworker stdout | 2025-11-04 09:06:54,958 [75] [DEBUG] [data.model.storage] Garbage collecting storage 28 namespacegcworker stdout | 2025-11-04 09:06:54,958 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,959 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:54,960 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:54,960 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:54,961 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:54,961 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:54,962 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993']) namespacegcworker stdout | 2025-11-04 09:06:54,962 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993']) namespacegcworker stdout | 2025-11-04 09:06:54,964 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,965 [75] [DEBUG] [data.model.storage] Removing sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,965 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,965 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7'} namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,966 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | fec003a036f8d9ebd42305c72e9c61cab35d1af912bcb6d8b6fdfd1688708b88 namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 852618753c3f99291ea6040e2f78e0b48fe2e120cd1af417c5ee68e460c9bb2a namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=852618753c3f99291ea6040e2f78e0b48fe2e120cd1af417c5ee68e460c9bb2a', 'amz-sdk-invocation-id': b'f43740b2-1c99-4835-8ca4-0470b0ea081c', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,967 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,971 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,971 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpiw-2b362b-7ip', 'x-amz-id-2': 'mhkchpiw-2b362b-7ip', 'ETag': '"5fd400f45e4dfcda15ec19cba6d9434d-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:47 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '4135941', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,971 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpiw-2b362b-7ip', 'HostId': 'mhkchpiw-2b362b-7ip', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpiw-2b362b-7ip', 'x-amz-id-2': 'mhkchpiw-2b362b-7ip', 'etag': '"5fd400f45e4dfcda15ec19cba6d9434d-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:47 GMT', 'content-type': 'application/octet-stream', 'content-length': '4135941', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 47, tzinfo=tzutc()), 'ContentLength': 4135941, 'ETag': '"5fd400f45e4dfcda15ec19cba6d9434d-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7'} namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,972 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | fc474fc2ec5fa5fda8a594bbda444eaa6fa10d476ea4159c8cf20268d60c8e32 namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 58771f889fddc5798ad9df745368818414165270418d10dd09ab8b20fdf9b1bd namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,973 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=58771f889fddc5798ad9df745368818414165270418d10dd09ab8b20fdf9b1bd', 'amz-sdk-invocation-id': b'b1b9d811-6157-41bc-ba4e-4c3ee207e133', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,974 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,979 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d6/d69d4d41cfe2ee680d6972795e2a1eb9e4dc4ec3b3c5e0797c9ab43bb3726fa7 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpj2-2eunre-kda', 'x-amz-id-2': 'mhkchpj2-2eunre-kda', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpj2-2eunre-kda', 'HostId': 'mhkchpj2-2eunre-kda', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpj2-2eunre-kda', 'x-amz-id-2': 'mhkchpj2-2eunre-kda', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,980 [75] [DEBUG] [data.model.storage] Removing sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,981 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483'} namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,982 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | abd36d6a54034c2fbfb858f717ed47d910c20875e8d0b7b13db14d8ca9817456 namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 95bd47c25f03ca588cc74fe9c57c700ce111eb9e06fb39006b78816accea3b13 namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=95bd47c25f03ca588cc74fe9c57c700ce111eb9e06fb39006b78816accea3b13', 'amz-sdk-invocation-id': b'a354e9da-1957-4983-82e3-3f29ffea399c', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,983 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjc-2kkcid-6q6', 'x-amz-id-2': 'mhkchpjc-2kkcid-6q6', 'ETag': '"13bd928752c06ba71e4750bf5a90b84d-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:49 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5749', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,987 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpjc-2kkcid-6q6', 'HostId': 'mhkchpjc-2kkcid-6q6', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjc-2kkcid-6q6', 'x-amz-id-2': 'mhkchpjc-2kkcid-6q6', 'etag': '"13bd928752c06ba71e4750bf5a90b84d-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:49 GMT', 'content-type': 'application/octet-stream', 'content-length': '5749', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 49, tzinfo=tzutc()), 'ContentLength': 5749, 'ETag': '"13bd928752c06ba71e4750bf5a90b84d-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483'} namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,988 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e2564197b10ccb2a705dc830b57c8b888133aaf7224cfb8353a773291aad6757 namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3be68753eb89ba4f3d2cc536f7ea8fce55aa1cf0ab81d3b29d589ba24d72dd18 namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3be68753eb89ba4f3d2cc536f7ea8fce55aa1cf0ab81d3b29d589ba24d72dd18', 'amz-sdk-invocation-id': b'03d21201-6bc4-4bbf-bc24-68df2dfd8f5f', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,989 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0b/0b4a39c2c94e3953aed34e96ce61e0414d2e50d1c469d68a20bc73239ebce483 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpji-2o0eru-1xx', 'x-amz-id-2': 'mhkchpji-2o0eru-1xx', 'Date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,995 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpji-2o0eru-1xx', 'HostId': 'mhkchpji-2o0eru-1xx', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpji-2o0eru-1xx', 'x-amz-id-2': 'mhkchpji-2o0eru-1xx', 'date': 'Tue, 04 Nov 2025 09:06:54 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:54,996 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 1]) namespacegcworker stdout | 2025-11-04 09:06:54,996 [75] [DEBUG] [data.model.storage] Removing sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07 from local_us namespacegcworker stdout | 2025-11-04 09:06:54,996 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07'} namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,997 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090654Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090654Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 8a694b516f67cffcd2a46bcc6230ba8845bcf28d4e89a2c219e1f87c16ab10f4 namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 9ce56560cedca41e7ae0ee4f76edcacb572b344580d63cceec82d0e9e9dbb012 namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:54,998 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:54,999 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090654Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ce56560cedca41e7ae0ee4f76edcacb572b344580d63cceec82d0e9e9dbb012', 'amz-sdk-invocation-id': b'fb2929c7-d4d2-438b-8e13-4f4ea4abfc92', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:54,999 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,002 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,002 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjs-2tss15-7kh', 'x-amz-id-2': 'mhkchpjs-2tss15-7kh', 'ETag': '"3121b88a3c6c2de99e86fe4710f17176-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:45 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,002 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpjs-2tss15-7kh', 'HostId': 'mhkchpjs-2tss15-7kh', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjs-2tss15-7kh', 'x-amz-id-2': 'mhkchpjs-2tss15-7kh', 'etag': '"3121b88a3c6c2de99e86fe4710f17176-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:45 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 45, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"3121b88a3c6c2de99e86fe4710f17176-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07'} namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,003 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | eb20f404019f521bb32509dd3db8846c05845064a396fd4e0cde0152ce5f99a4 namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | e4aa973164dc53094a3b1b72b2b30fd9b64b0ce758526d82a1c00bb42985f693 namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e4aa973164dc53094a3b1b72b2b30fd9b64b0ce758526d82a1c00bb42985f693', 'amz-sdk-invocation-id': b'311be6f2-85bc-4c79-aaf0-3984738633c2', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,004 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,010 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/48/48524b92080ff6c6e895f280246e59de8de80ee86754a5fd012d7cc6ff4f8e07 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjx-2xbeqx-1114', 'x-amz-id-2': 'mhkchpjx-2xbeqx-1114', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpjx-2xbeqx-1114', 'HostId': 'mhkchpjx-2xbeqx-1114', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpjx-2xbeqx-1114', 'x-amz-id-2': 'mhkchpjx-2xbeqx-1114', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,011 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [data.model.storage] Removing sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993'} namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,012 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | bfbfea96831689c8f390cbe56d0290aabbdfdb85a49073012e97a8ccd0ae975c namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 284b45fb5aa61ba83239ed2ed473dd0e6b1a7350ec559c65f813975a0be7442a namespacegcworker stdout | 2025-11-04 09:06:55,013 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,014 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,014 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,014 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=284b45fb5aa61ba83239ed2ed473dd0e6b1a7350ec559c65f813975a0be7442a', 'amz-sdk-invocation-id': b'425d5ddb-0817-4d9c-a1a5-701a67ad2ec9', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,014 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,017 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,017 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpk7-32ty7m-v8o', 'x-amz-id-2': 'mhkchpk7-32ty7m-v8o', 'ETag': '"d972e2208679bf56cb345564cf1d854b-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:52 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3616029', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,017 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpk7-32ty7m-v8o', 'HostId': 'mhkchpk7-32ty7m-v8o', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpk7-32ty7m-v8o', 'x-amz-id-2': 'mhkchpk7-32ty7m-v8o', 'etag': '"d972e2208679bf56cb345564cf1d854b-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:52 GMT', 'content-type': 'application/octet-stream', 'content-length': '3616029', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 52, tzinfo=tzutc()), 'ContentLength': 3616029, 'ETag': '"d972e2208679bf56cb345564cf1d854b-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993'} namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,018 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 8888bbfb9b772c7364258cac4c1a0985c44b9e6b34a693dfbf0dca4c6b65dab8 namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6de56239f63c4357007d9275e7a638ae5385b0cb2c20c9fb62ddd9f81f432778 namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6de56239f63c4357007d9275e7a638ae5385b0cb2c20c9fb62ddd9f81f432778', 'amz-sdk-invocation-id': b'21f8c3ac-d6dc-4403-ae86-f2f4ad4b9101', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,019 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c7/c787620501b746b3aa9ec021f3ddb0b707572b5c68e33d73be392b9c078a4993 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkc-361vto-1tg', 'x-amz-id-2': 'mhkchpkc-361vto-1tg', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpkc-361vto-1tg', 'HostId': 'mhkchpkc-361vto-1tg', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkc-361vto-1tg', 'x-amz-id-2': 'mhkchpkc-361vto-1tg', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,026 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,027 [75] [DEBUG] [data.model.storage] Removing sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a from local_us namespacegcworker stdout | 2025-11-04 09:06:55,027 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a'} namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,028 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | d8f78878802c8e8305e3d458b902ace22a72467371e19d9792232b25492ab4b5 namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 419ca8fbf8a67e4ea8b0e6b001dc4881f41868df620a558dbb61bf0a5d304812 namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,029 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,030 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=419ca8fbf8a67e4ea8b0e6b001dc4881f41868df620a558dbb61bf0a5d304812', 'amz-sdk-invocation-id': b'8154a45f-474b-4210-9778-71f6f1772ecb', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,030 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,033 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,034 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkn-3cajrn-n3v', 'x-amz-id-2': 'mhkchpkn-3cajrn-n3v', 'ETag': '"aa8f171de98f25e9144e08ebfa6bca6d-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:48 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '597', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,034 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,034 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpkn-3cajrn-n3v', 'HostId': 'mhkchpkn-3cajrn-n3v', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkn-3cajrn-n3v', 'x-amz-id-2': 'mhkchpkn-3cajrn-n3v', 'etag': '"aa8f171de98f25e9144e08ebfa6bca6d-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:48 GMT', 'content-type': 'application/octet-stream', 'content-length': '597', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 48, tzinfo=tzutc()), 'ContentLength': 597, 'ETag': '"aa8f171de98f25e9144e08ebfa6bca6d-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a'} namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,035 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1260e19f62721dd1970829937a730caa68b90d8ff53aeacefe2f07bb297f37b5 namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6a98acd6d3cfe851879df8762c577737e6b472b9dcd7b2b1c17bcd1dfc1c1627 namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6a98acd6d3cfe851879df8762c577737e6b472b9dcd7b2b1c17bcd1dfc1c1627', 'amz-sdk-invocation-id': b'980da5e4-9bb5-4904-99f6-1bec7e536a05', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,036 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,043 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2abc5e83407155714f171c293f197e1310176959e106f8ad63ffa2e7e9635d4a HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,043 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkt-3g6qu2-174g', 'x-amz-id-2': 'mhkchpkt-3g6qu2-174g', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpkt-3g6qu2-174g', 'HostId': 'mhkchpkt-3g6qu2-174g', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpkt-3g6qu2-174g', 'x-amz-id-2': 'mhkchpkt-3g6qu2-174g', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [data.model.storage] Removing sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,044 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4'} namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,045 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2c86a6218580820ff50eaa284877ac2d7a9cbb109033243f0fc54eb0ac01fd0b namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1ea54250000d0b5e4ab7344409cd670579ec6dab540271f393c1557c18173086 namespacegcworker stdout | 2025-11-04 09:06:55,046 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,047 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,047 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,047 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1ea54250000d0b5e4ab7344409cd670579ec6dab540271f393c1557c18173086', 'amz-sdk-invocation-id': b'02037a99-f426-4b3d-a02f-2007fba9cbc4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,047 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,051 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,051 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpl4-3mo3ww-ck', 'x-amz-id-2': 'mhkchpl4-3mo3ww-ck', 'ETag': '"3f4dc804a3bdcd9477d2ca35497b09c9-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:50 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '81012', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,051 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpl4-3mo3ww-ck', 'HostId': 'mhkchpl4-3mo3ww-ck', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpl4-3mo3ww-ck', 'x-amz-id-2': 'mhkchpl4-3mo3ww-ck', 'etag': '"3f4dc804a3bdcd9477d2ca35497b09c9-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:50 GMT', 'content-type': 'application/octet-stream', 'content-length': '81012', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 50, tzinfo=tzutc()), 'ContentLength': 81012, 'ETag': '"3f4dc804a3bdcd9477d2ca35497b09c9-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4'} namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,052 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6a095512be45ab1c8b612fe7a1b8794ade88d5b56e9d3a9e20403af4a125a4ea namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | e7252bc7558741aecf9d2507946366007c43388eba4e35f7f96c4b6c7480a7ad namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e7252bc7558741aecf9d2507946366007c43388eba4e35f7f96c4b6c7480a7ad', 'amz-sdk-invocation-id': b'9746c054-abb6-461a-b731-a6645495f985', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,053 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,059 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d4/d405333e75d435c8468c7783fb12c5d4b72dc8e61369e4a584f5fe26cf4a63b4 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpla-3qcfru-iqw', 'x-amz-id-2': 'mhkchpla-3qcfru-iqw', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpla-3qcfru-iqw', 'HostId': 'mhkchpla-3qcfru-iqw', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpla-3qcfru-iqw', 'x-amz-id-2': 'mhkchpla-3qcfru-iqw', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,060 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,061 [75] [DEBUG] [data.model.storage] Removing sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,061 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,061 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283'} namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,062 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4783795f7080cb36825e6fbeb20e27a8875b8a3992ee6b19e70b47f209c10bd6 namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | fc18a4d81614195d18e16415313bf32c8cf6f40927e7c4879bd3de7fb322e964 namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=fc18a4d81614195d18e16415313bf32c8cf6f40927e7c4879bd3de7fb322e964', 'amz-sdk-invocation-id': b'b33c86f6-f112-4794-ab93-846328e6ef2f', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,063 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-web stdout | 2025-11-04 09:06:55,066 [261] [DEBUG] [app] Starting request: urn:request:c0712014-6de0-417f-acec-fc960644012f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchplk-3w668n-vm', 'x-amz-id-2': 'mhkchplk-3w668n-vm', 'ETag': '"ff5f07fe308df34f73245cf8878c3b21-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:50 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' gunicorn-web stdout | 2025-11-04 09:06:55,067 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,067 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchplk-3w668n-vm', 'HostId': 'mhkchplk-3w668n-vm', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchplk-3w668n-vm', 'x-amz-id-2': 'mhkchplk-3w668n-vm', 'etag': '"ff5f07fe308df34f73245cf8878c3b21-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:50 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 50, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"ff5f07fe308df34f73245cf8878c3b21-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283'} namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,068 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | c6b6491d32eddeb1466e7170183f66369fc2f8c58ca3db41e7c47d7ac796e91b namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | ea9874c234a8b99d241624005add69549e777ea65ee5f0430047568511af417f namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,069 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ea9874c234a8b99d241624005add69549e777ea65ee5f0430047568511af417f', 'amz-sdk-invocation-id': b'0e293364-06bd-4818-b8fa-2b72157b7106', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,070 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:06:55,073 [250] [DEBUG] [app] Starting request: urn:request:19bd6065-374a-407a-8f17-9dbd4bc2465a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:55,073 [250] [DEBUG] [app] Ending request: urn:request:19bd6065-374a-407a-8f17-9dbd4bc2465a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:19bd6065-374a-407a-8f17-9dbd4bc2465a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:06:55,074 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:55,074 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:55,075 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost namespacegcworker stdout | 2025-11-04 09:06:55,075 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5c/5c40dfa7b532e721f94b2701115434a034c5cece1152b87330140e4788b29283 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchplq-400rv3-1cej', 'x-amz-id-2': 'mhkchplq-400rv3-1cej', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchplq-400rv3-1cej', 'HostId': 'mhkchplq-400rv3-1cej', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchplq-400rv3-1cej', 'x-amz-id-2': 'mhkchplq-400rv3-1cej', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,076 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 1]) gunicorn-web stdout | 2025-11-04 09:06:55,076 [258] [DEBUG] [app] Starting request: urn:request:7b373a0b-53ab-4742-808c-46d8ae67a34f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:55,077 [258] [DEBUG] [app] Ending request: urn:request:7b373a0b-53ab-4742-808c-46d8ae67a34f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:7b373a0b-53ab-4742-808c-46d8ae67a34f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:55,077 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:06:55,077 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:55,077 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:55,077 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:55,077 [261] [INFO] [data.database] Connection pooling disabled for postgresql namespacegcworker stdout | 2025-11-04 09:06:55,078 [75] [DEBUG] [data.model.storage] Removing sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,078 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577'} namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,079 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,080 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1aec83120d6cbf172a3a706702d67af81171553129d18be806c88b024fa4c491 namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | fd3be08081ed32025d5536fb17007a6cd439c8d55e53be025331e2684b3a9f92 namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=fd3be08081ed32025d5536fb17007a6cd439c8d55e53be025331e2684b3a9f92', 'amz-sdk-invocation-id': b'78c5034c-4b76-4042-a676-d18a7351fce5', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,081 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-web stdout | 2025-11-04 09:06:55,082 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:55,083 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) namespacegcworker stdout | 2025-11-04 09:06:55,085 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,085 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpm2-474b8v-svu', 'x-amz-id-2': 'mhkchpm2-474b8v-svu', 'ETag': '"b56804647948f534a29bd63db3e05be4-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:42 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '593', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,085 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpm2-474b8v-svu', 'HostId': 'mhkchpm2-474b8v-svu', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpm2-474b8v-svu', 'x-amz-id-2': 'mhkchpm2-474b8v-svu', 'etag': '"b56804647948f534a29bd63db3e05be4-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:42 GMT', 'content-type': 'application/octet-stream', 'content-length': '593', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 42, tzinfo=tzutc()), 'ContentLength': 593, 'ETag': '"b56804647948f534a29bd63db3e05be4-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577'} namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,086 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-web stdout | 2025-11-04 09:06:55,087 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0a0ba312b4602bc88d147578f1ece8f4c1044f998055ee3e982103efd18883b2 namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 8a47951871508ca1de46306e379f24807d0be7e724932efe4487c1ee55e184c9 namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,087 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,088 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,088 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8a47951871508ca1de46306e379f24807d0be7e724932efe4487c1ee55e184c9', 'amz-sdk-invocation-id': b'f11aed3f-470b-4eeb-8417-9d3536a07622', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,088 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-web stdout | 2025-11-04 09:06:55,088 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:55,089 [261] [DEBUG] [app] Ending request: urn:request:c0712014-6de0-417f-acec-fc960644012f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c0712014-6de0-417f-acec-fc960644012f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:55,089 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:55,090 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/1717b9d509321d1c04cfcc2c1c9653e1556e8b87e34a532d0f5c3bdbcf501577 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpm9-4avrry-1tv', 'x-amz-id-2': 'mhkchpm9-4avrry-1tv', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpm9-4avrry-1tv', 'HostId': 'mhkchpm9-4avrry-1tv', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpm9-4avrry-1tv', 'x-amz-id-2': 'mhkchpm9-4avrry-1tv', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,094 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,095 [75] [DEBUG] [data.model.storage] Removing sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,095 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,095 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1'} namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,096 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | af362ffcac7f4a65407bb4afe45d60c1c846626044dbc771c58a972a105d0527 namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6ca0ba262425017639706f0cbb41dc518645392012f5a4eb26c3eb4c15edacd5 namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6ca0ba262425017639706f0cbb41dc518645392012f5a4eb26c3eb4c15edacd5', 'amz-sdk-invocation-id': b'8ba934e6-93c8-4597-b10c-c977bd040c55', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,097 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,101 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,101 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpmi-4ghkii-ifo', 'x-amz-id-2': 'mhkchpmi-4ghkii-ifo', 'ETag': '"7b63811f71804eeae2341355191851fc-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:44 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5719', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,101 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,101 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpmi-4ghkii-ifo', 'HostId': 'mhkchpmi-4ghkii-ifo', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpmi-4ghkii-ifo', 'x-amz-id-2': 'mhkchpmi-4ghkii-ifo', 'etag': '"7b63811f71804eeae2341355191851fc-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:44 GMT', 'content-type': 'application/octet-stream', 'content-length': '5719', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 44, tzinfo=tzutc()), 'ContentLength': 5719, 'ETag': '"7b63811f71804eeae2341355191851fc-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1'} namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,102 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,103 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,103 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,103 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,103 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 5cc88775345bdc4ce46537ebf3ff83e07bbdaecc7c0024312ac1e7910b8a5b11 namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 24c09a5b3e4cfd4b1bb92a4c300e61c56e43ab4cf78293502405af97d920d6b5 namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=24c09a5b3e4cfd4b1bb92a4c300e61c56e43ab4cf78293502405af97d920d6b5', 'amz-sdk-invocation-id': b'cde49683-04cf-4f18-8dbc-90ccf7361ad1', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,104 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,110 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/90/90d9ddc482be902a6ac1f0d27326597273595d46c7557771af2f3bd8b90853d1 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpmp-4ktmbq-kvd', 'x-amz-id-2': 'mhkchpmp-4ktmbq-kvd', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpmp-4ktmbq-kvd', 'HostId': 'mhkchpmp-4ktmbq-kvd', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpmp-4ktmbq-kvd', 'x-amz-id-2': 'mhkchpmp-4ktmbq-kvd', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,111 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,112 [75] [DEBUG] [data.model.storage] Removing sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba from local_us namespacegcworker stdout | 2025-11-04 09:06:55,112 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba'} namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,113 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4783b710d67ba52d87f4e213ff8a9cde2492b47bea4580ad9b0097a8e71e8438 namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 673a7b0db688f665f7a98325db0f6580a828cf696210f75d7e95c3c747008b61 namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,114 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,115 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=673a7b0db688f665f7a98325db0f6580a828cf696210f75d7e95c3c747008b61', 'amz-sdk-invocation-id': b'b5bb7e19-955d-4f57-9457-aba78ceadafd', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,115 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,118 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,118 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpn0-4qxoqk-7c3', 'x-amz-id-2': 'mhkchpn0-4qxoqk-7c3', 'ETag': '"247c002cb45736cc4164ca1fefb164e0-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:44 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80992', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,118 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpn0-4qxoqk-7c3', 'HostId': 'mhkchpn0-4qxoqk-7c3', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpn0-4qxoqk-7c3', 'x-amz-id-2': 'mhkchpn0-4qxoqk-7c3', 'etag': '"247c002cb45736cc4164ca1fefb164e0-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:44 GMT', 'content-type': 'application/octet-stream', 'content-length': '80992', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 44, tzinfo=tzutc()), 'ContentLength': 80992, 'ETag': '"247c002cb45736cc4164ca1fefb164e0-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba'} namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,119 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2d4523298febe011211f516e431e2d1b0633088f73c9a37c760187958c6fcffb namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 19bead327db61fae7d05f981d43df541cc7e3ad5ed6cb9912b0ff7a00422fbdd namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,120 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=19bead327db61fae7d05f981d43df541cc7e3ad5ed6cb9912b0ff7a00422fbdd', 'amz-sdk-invocation-id': b'549c7800-b035-4239-ad41-0a420d2d3f84', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,121 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-web stdout | 2025-11-04 09:06:55,122 [261] [DEBUG] [app] Starting request: urn:request:42175b65-cae5-4ac3-95d2-8a969e789428 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:06:55,124 [250] [DEBUG] [app] Starting request: urn:request:0f2e8a8c-862d-4d83-9954-4df21c91bf85 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:06:55,124 [250] [DEBUG] [app] Ending request: urn:request:0f2e8a8c-862d-4d83-9954-4df21c91bf85 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0f2e8a8c-862d-4d83-9954-4df21c91bf85', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:06:55,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:06:55,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:55,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/30/3032b259f8109e3849ca23c23e8a8e9749a57d7ec26c7d53181220442b7fd7ba HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpn5-4uelwr-n3l', 'x-amz-id-2': 'mhkchpn5-4uelwr-n3l', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,126 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpn5-4uelwr-n3l', 'HostId': 'mhkchpn5-4uelwr-n3l', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpn5-4uelwr-n3l', 'x-amz-id-2': 'mhkchpn5-4uelwr-n3l', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,127 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) gunicorn-web stdout | 2025-11-04 09:06:55,127 [261] [DEBUG] [app] Starting request: urn:request:1e8f2464-3e43-4558-8a66-5c7332762190 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:06:55,127 [261] [DEBUG] [app] Ending request: urn:request:1e8f2464-3e43-4558-8a66-5c7332762190 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1e8f2464-3e43-4558-8a66-5c7332762190', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:06:55,128 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:06:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" namespacegcworker stdout | 2025-11-04 09:06:55,128 [75] [DEBUG] [data.model.gc] Found 10 uploaded blobs to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,128 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 28 under repository 7 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:06:55,128 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:06:55,128 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:06:55,128 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:06:55,128 [261] [INFO] [data.database] Connection pooling disabled for postgresql namespacegcworker stdout | 2025-11-04 09:06:55,128 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,129 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 29 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,130 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:55,131 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 30 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,131 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [30]) gunicorn-web stdout | 2025-11-04 09:06:55,132 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:06:55,132 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) namespacegcworker stdout | 2025-11-04 09:06:55,133 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 31 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,133 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:55,134 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 32 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,134 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:55,136 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 33 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,136 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [33]) gunicorn-web stdout | 2025-11-04 09:06:55,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) namespacegcworker stdout | 2025-11-04 09:06:55,137 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 34 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,138 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [34]) gunicorn-web stdout | 2025-11-04 09:06:55,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:06:55,139 [261] [DEBUG] [app] Ending request: urn:request:42175b65-cae5-4ac3-95d2-8a969e789428 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:42175b65-cae5-4ac3-95d2-8a969e789428', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:06:55,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:06:55,140 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:06:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" namespacegcworker stdout | 2025-11-04 09:06:55,139 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 35 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,140 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [35]) nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:06:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) namespacegcworker stdout | 2025-11-04 09:06:55,141 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 36 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,141 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:55,142 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 37 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,143 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:55,144 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {32, 33, 34, 35, 36, 37, 38, 29, 30, 31} namespacegcworker stdout | 2025-11-04 09:06:55,144 [75] [DEBUG] [data.model.storage] Garbage collecting storage 32 namespacegcworker stdout | 2025-11-04 09:06:55,144 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,145 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [32, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,146 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:55,146 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:55,147 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:55,147 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [32]) namespacegcworker stdout | 2025-11-04 09:06:55,148 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095']) namespacegcworker stdout | 2025-11-04 09:06:55,148 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095']) namespacegcworker stdout | 2025-11-04 09:06:55,150 [75] [DEBUG] [data.model.storage] Garbage collecting storage 33 namespacegcworker stdout | 2025-11-04 09:06:55,150 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,151 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [33, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,152 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [33]) namespacegcworker stdout | 2025-11-04 09:06:55,152 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [33]) namespacegcworker stdout | 2025-11-04 09:06:55,152 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [33]) namespacegcworker stdout | 2025-11-04 09:06:55,153 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [33]) namespacegcworker stdout | 2025-11-04 09:06:55,153 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0']) namespacegcworker stdout | 2025-11-04 09:06:55,154 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0']) namespacegcworker stdout | 2025-11-04 09:06:55,155 [75] [DEBUG] [data.model.storage] Garbage collecting storage 34 namespacegcworker stdout | 2025-11-04 09:06:55,155 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,156 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [34, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,157 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [34]) namespacegcworker stdout | 2025-11-04 09:06:55,157 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [34]) namespacegcworker stdout | 2025-11-04 09:06:55,157 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [34]) namespacegcworker stdout | 2025-11-04 09:06:55,158 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [34]) namespacegcworker stdout | 2025-11-04 09:06:55,158 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee']) namespacegcworker stdout | 2025-11-04 09:06:55,159 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee']) namespacegcworker stdout | 2025-11-04 09:06:55,160 [75] [DEBUG] [data.model.storage] Garbage collecting storage 35 namespacegcworker stdout | 2025-11-04 09:06:55,160 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,161 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [35, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,161 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [35]) namespacegcworker stdout | 2025-11-04 09:06:55,162 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [35]) namespacegcworker stdout | 2025-11-04 09:06:55,162 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [35]) namespacegcworker stdout | 2025-11-04 09:06:55,162 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [35]) namespacegcworker stdout | 2025-11-04 09:06:55,163 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60']) namespacegcworker stdout | 2025-11-04 09:06:55,164 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60']) namespacegcworker stdout | 2025-11-04 09:06:55,165 [75] [DEBUG] [data.model.storage] Garbage collecting storage 36 namespacegcworker stdout | 2025-11-04 09:06:55,165 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,166 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [36, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,167 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:55,167 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:55,168 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:55,168 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [36]) namespacegcworker stdout | 2025-11-04 09:06:55,168 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e']) namespacegcworker stdout | 2025-11-04 09:06:55,169 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e']) namespacegcworker stdout | 2025-11-04 09:06:55,171 [75] [DEBUG] [data.model.storage] Garbage collecting storage 37 namespacegcworker stdout | 2025-11-04 09:06:55,171 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,171 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [37, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,172 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:55,173 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:55,173 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:55,173 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [37]) namespacegcworker stdout | 2025-11-04 09:06:55,174 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e']) namespacegcworker stdout | 2025-11-04 09:06:55,174 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e']) namespacegcworker stdout | 2025-11-04 09:06:55,175 [75] [DEBUG] [data.model.storage] Garbage collecting storage 38 namespacegcworker stdout | 2025-11-04 09:06:55,176 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,176 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [38, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,177 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:55,177 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:55,178 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:55,178 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:55,179 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39']) namespacegcworker stdout | 2025-11-04 09:06:55,180 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39']) namespacegcworker stdout | 2025-11-04 09:06:55,181 [75] [DEBUG] [data.model.storage] Garbage collecting storage 29 namespacegcworker stdout | 2025-11-04 09:06:55,181 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,182 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,182 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:55,183 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:55,183 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:55,184 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:55,184 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c']) namespacegcworker stdout | 2025-11-04 09:06:55,185 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c']) namespacegcworker stdout | 2025-11-04 09:06:55,186 [75] [DEBUG] [data.model.storage] Garbage collecting storage 30 namespacegcworker stdout | 2025-11-04 09:06:55,186 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,187 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,188 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:55,188 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:55,189 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:55,189 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:55,189 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac']) namespacegcworker stdout | 2025-11-04 09:06:55,190 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac']) namespacegcworker stdout | 2025-11-04 09:06:55,191 [75] [DEBUG] [data.model.storage] Garbage collecting storage 31 namespacegcworker stdout | 2025-11-04 09:06:55,191 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,192 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [31, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,193 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:55,193 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:55,193 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:55,194 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [31]) namespacegcworker stdout | 2025-11-04 09:06:55,194 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e']) namespacegcworker stdout | 2025-11-04 09:06:55,195 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e']) namespacegcworker stdout | 2025-11-04 09:06:55,196 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,197 [75] [DEBUG] [data.model.storage] Removing sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,197 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0'} namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,198 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4df5155aba28cb4224dcbe3e106dacf1fabc7b931e9144787ca6b241a1abd033 namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 4dc6041f2be57ed0bbf5ac6b7d3969e31cf2dcdac85711248bfbd9a936f2abe5 namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,199 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,200 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,200 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4dc6041f2be57ed0bbf5ac6b7d3969e31cf2dcdac85711248bfbd9a936f2abe5', 'amz-sdk-invocation-id': b'10d64f64-9611-4489-a846-e3fde4d8327a', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,200 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppd-65mlp6-1aga', 'x-amz-id-2': 'mhkchppd-65mlp6-1aga', 'ETag': '"5a0c70292ce44f1fe38afbc820d789ec-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:58 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3730187', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,205 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchppd-65mlp6-1aga', 'HostId': 'mhkchppd-65mlp6-1aga', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppd-65mlp6-1aga', 'x-amz-id-2': 'mhkchppd-65mlp6-1aga', 'etag': '"5a0c70292ce44f1fe38afbc820d789ec-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:58 GMT', 'content-type': 'application/octet-stream', 'content-length': '3730187', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 58, tzinfo=tzutc()), 'ContentLength': 3730187, 'ETag': '"5a0c70292ce44f1fe38afbc820d789ec-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0'} namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,206 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | bf13f2346e6ef5ed2e24fae66e3b22d2e6e2823460ffd9d6f6ab911dd185f6e4 namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6ccaa162b38752b8364bf15fc510cefdb9b7e14642cf1046965ed2ecab3d44d6 namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6ccaa162b38752b8364bf15fc510cefdb9b7e14642cf1046965ed2ecab3d44d6', 'amz-sdk-invocation-id': b'deb15a90-19b2-4163-be2c-2bfbea107a65', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,207 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,213 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/33/33a2433d89df7e794d1655fce70d7031d8065c9798bdc2931f7c98fcc8d310d0 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,213 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppk-69tqhp-1dxy', 'x-amz-id-2': 'mhkchppk-69tqhp-1dxy', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,213 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,213 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchppk-69tqhp-1dxy', 'HostId': 'mhkchppk-69tqhp-1dxy', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppk-69tqhp-1dxy', 'x-amz-id-2': 'mhkchppk-69tqhp-1dxy', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [data.model.storage] Removing sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,214 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095'} namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,215 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,216 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b15e9166c8bd77a2d16d707949de4fae4d7ce5b7d93a0445ae156a8b9c784a57 namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 64d9866338517c819325524a2a2e7149d8a2ad15da65307bbdbe50745101a53f namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=64d9866338517c819325524a2a2e7149d8a2ad15da65307bbdbe50745101a53f', 'amz-sdk-invocation-id': b'49483b9e-0419-4f01-a88a-870002d22213', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,217 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,220 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppu-6ftetz-zu6', 'x-amz-id-2': 'mhkchppu-6ftetz-zu6', 'ETag': '"61a75d0e757232a88592a29f55d878e0-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:56 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchppu-6ftetz-zu6', 'HostId': 'mhkchppu-6ftetz-zu6', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchppu-6ftetz-zu6', 'x-amz-id-2': 'mhkchppu-6ftetz-zu6', 'etag': '"61a75d0e757232a88592a29f55d878e0-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:56 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 56, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"61a75d0e757232a88592a29f55d878e0-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095'} namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,221 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 3f2c8c914d4f09f5ec13817cf8fb0acf71aa023bd73793cd227c09b45fdd623f namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 478752f87e3146ded2cd467ff00a17405f7403c3acdb04ba1cbe4cbcd3a7ea95 namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,222 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,223 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=478752f87e3146ded2cd467ff00a17405f7403c3acdb04ba1cbe4cbcd3a7ea95', 'amz-sdk-invocation-id': b'493efc30-108d-409b-88f7-28f41002f6f0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,223 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/76/761452f61cc7faecf79a125b3ce0ea53f36fc7dd97a263963e2659f25fe4a095 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpq0-6jejf6-540', 'x-amz-id-2': 'mhkchpq0-6jejf6-540', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,229 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpq0-6jejf6-540', 'HostId': 'mhkchpq0-6jejf6-540', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpq0-6jejf6-540', 'x-amz-id-2': 'mhkchpq0-6jejf6-540', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,230 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,230 [75] [DEBUG] [data.model.storage] Removing sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac from local_us namespacegcworker stdout | 2025-11-04 09:06:55,230 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac'} namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,231 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 9b26bc0e3ab5a5eb766cefe719aa928fcf3bb613de30ee4d86f3f5d30d10ec55 namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | e2b494b5762e9aa011ac8ec9fd5f307efcfe17602516fb1a409e60938a16d428 namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e2b494b5762e9aa011ac8ec9fd5f307efcfe17602516fb1a409e60938a16d428', 'amz-sdk-invocation-id': b'89c84e6a-a801-4e9a-9420-4e64a61f1ac4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,232 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpq9-6oxja0-tke', 'x-amz-id-2': 'mhkchpq9-6oxja0-tke', 'ETag': '"9c5b9ae38c3793dc8e458e2508cbe466-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:55 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80931', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpq9-6oxja0-tke', 'HostId': 'mhkchpq9-6oxja0-tke', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpq9-6oxja0-tke', 'x-amz-id-2': 'mhkchpq9-6oxja0-tke', 'etag': '"9c5b9ae38c3793dc8e458e2508cbe466-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:55 GMT', 'content-type': 'application/octet-stream', 'content-length': '80931', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 55, tzinfo=tzutc()), 'ContentLength': 80931, 'ETag': '"9c5b9ae38c3793dc8e458e2508cbe466-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac'} namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,236 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1e71c7f8ae806ea5e919891365a3c25335c98bdc287306e43ba1839b459f0231 namespacegcworker stdout | 2025-11-04 09:06:55,237 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | c31e15368a223ce02ace6fd7de62634f7cec5e56acc53c3b2330e8e0a9537920 namespacegcworker stdout | 2025-11-04 09:06:55,238 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,238 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,238 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,238 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c31e15368a223ce02ace6fd7de62634f7cec5e56acc53c3b2330e8e0a9537920', 'amz-sdk-invocation-id': b'1e2fe19f-31ad-42a7-9acd-539ffd43fcb0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,238 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/b1/b1c2aed2f4fbd5edc68212bb46bacd85cabf9ebec62b30efec7b7657f0213eac HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqf-6s8ljv-c0n', 'x-amz-id-2': 'mhkchpqf-6s8ljv-c0n', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,244 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpqf-6s8ljv-c0n', 'HostId': 'mhkchpqf-6s8ljv-c0n', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqf-6s8ljv-c0n', 'x-amz-id-2': 'mhkchpqf-6s8ljv-c0n', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,245 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,245 [75] [DEBUG] [data.model.storage] Removing sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c from local_us namespacegcworker stdout | 2025-11-04 09:06:55,245 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c'} namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,246 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,247 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,247 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,247 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | a990e927e5681515e62634b463ba8794dd0a049c8cbc7dbdce66ed495d5d4c18 namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | aeff778ebacda26f3747a0fd219f27dbd98be030a0e2cba334460a0cb012eb80 namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,248 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,249 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,249 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=aeff778ebacda26f3747a0fd219f27dbd98be030a0e2cba334460a0cb012eb80', 'amz-sdk-invocation-id': b'08235cb1-338e-4db6-8e79-2a1d550ad08b', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,249 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,252 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,252 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqq-6yrjnu-fei', 'x-amz-id-2': 'mhkchpqq-6yrjnu-fei', 'ETag': '"ad3471c562bc7fd7ea9b64d2c5eaf89d-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:53 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '576', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,252 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpqq-6yrjnu-fei', 'HostId': 'mhkchpqq-6yrjnu-fei', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqq-6yrjnu-fei', 'x-amz-id-2': 'mhkchpqq-6yrjnu-fei', 'etag': '"ad3471c562bc7fd7ea9b64d2c5eaf89d-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:53 GMT', 'content-type': 'application/octet-stream', 'content-length': '576', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 53, tzinfo=tzutc()), 'ContentLength': 576, 'ETag': '"ad3471c562bc7fd7ea9b64d2c5eaf89d-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c'} namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,253 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1d237a2a1e245ee0bf8249ca5e23744aa8d87cb3bd4c6f542a28442fc34e1b17 namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1028919d3724da8e2870ab4986421f3d7ff935e157eb47db3f6b110296c0983a namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,254 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1028919d3724da8e2870ab4986421f3d7ff935e157eb47db3f6b110296c0983a', 'amz-sdk-invocation-id': b'1b7d7741-cf02-4a8f-9546-29f24f5624d8', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,255 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/95/957b6b57f4173ba3ebfc2b2582d0ad72d325265f0ac7c1fe20356c855c4a955c HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqw-726xhn-vn4', 'x-amz-id-2': 'mhkchpqw-726xhn-vn4', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,260 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpqw-726xhn-vn4', 'HostId': 'mhkchpqw-726xhn-vn4', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpqw-726xhn-vn4', 'x-amz-id-2': 'mhkchpqw-726xhn-vn4', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,261 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,262 [75] [DEBUG] [data.model.storage] Removing sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e from local_us namespacegcworker stdout | 2025-11-04 09:06:55,262 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e'} namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,263 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,264 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | fd8196db9c246ece47b63e7c374e5ecd37a12b7267120883d3a998c6495667bc namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 701d5a24d24d3fb0cdd25a29e29d4ca32a9e48d4c9ebb8dc02b38a234b8fe990 namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=701d5a24d24d3fb0cdd25a29e29d4ca32a9e48d4c9ebb8dc02b38a234b8fe990', 'amz-sdk-invocation-id': b'dc528c57-0a19-4c38-929a-e8c8e1edc996', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,265 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpr6-78d1v2-ufs', 'x-amz-id-2': 'mhkchpr6-78d1v2-ufs', 'ETag': '"517b0a2657a6f159dbb0c68ae54f1307-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:03 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,269 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpr6-78d1v2-ufs', 'HostId': 'mhkchpr6-78d1v2-ufs', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpr6-78d1v2-ufs', 'x-amz-id-2': 'mhkchpr6-78d1v2-ufs', 'etag': '"517b0a2657a6f159dbb0c68ae54f1307-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:03 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 3, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"517b0a2657a6f159dbb0c68ae54f1307-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e'} namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,270 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 17b42175bd95bc9a2e4c7740eb6c9cb565285af6cc16b1d8464c943df7995857 namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5e33c2cc2bfae801d123e9af01a7393ccc0f63f3231b2d2415d32e632d0f31c7 namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5e33c2cc2bfae801d123e9af01a7393ccc0f63f3231b2d2415d32e632d0f31c7', 'amz-sdk-invocation-id': b'8100b76f-d6f7-4f03-944f-ed4895a20a9b', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,271 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,276 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a1/a1a305e7cc0c7bd645c6cc51bc78c1a1ce2b2644dd116b0ddb5db45ce3816b9e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprc-7bxlcz-19al', 'x-amz-id-2': 'mhkchprc-7bxlcz-19al', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchprc-7bxlcz-19al', 'HostId': 'mhkchprc-7bxlcz-19al', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprc-7bxlcz-19al', 'x-amz-id-2': 'mhkchprc-7bxlcz-19al', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,277 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,278 [75] [DEBUG] [data.model.storage] Removing sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,278 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,278 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39'} namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,279 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6cc70917f3744ff8c8d26d6bc2160de4e4a057de0cf5d7f9d11a492f503b8502 namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 44bfffe35f936ab9c5c20602ade1d4fb260cbba55b0b73bd010826e871c948c9 namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=44bfffe35f936ab9c5c20602ade1d4fb260cbba55b0b73bd010826e871c948c9', 'amz-sdk-invocation-id': b'de6f95e6-a604-49d0-b53b-e8904468b7f5', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,280 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,283 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,283 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprl-7han4p-cln', 'x-amz-id-2': 'mhkchprl-7han4p-cln', 'ETag': '"db7c07c311eb71db7cb910162089cda1-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:04 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3513811', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,283 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchprl-7han4p-cln', 'HostId': 'mhkchprl-7han4p-cln', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprl-7han4p-cln', 'x-amz-id-2': 'mhkchprl-7han4p-cln', 'etag': '"db7c07c311eb71db7cb910162089cda1-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:04 GMT', 'content-type': 'application/octet-stream', 'content-length': '3513811', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 4, tzinfo=tzutc()), 'ContentLength': 3513811, 'ETag': '"db7c07c311eb71db7cb910162089cda1-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39'} namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,284 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2f2866d1ade6b44237b2720992450ce8b2ccc41007ad50b165c28c38e41d8e53 namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3a83eed5027cbba2342c2bc8d079916e1f80977e739db805047454cd93495c18 namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3a83eed5027cbba2342c2bc8d079916e1f80977e739db805047454cd93495c18', 'amz-sdk-invocation-id': b'8e7edb76-c580-470a-b0dd-5ca6c3f34345', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,285 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,291 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a0/a0a4beaa1960bba353066d674aa0e3378b856b09e6d3f704d1755daa5d6c1d39 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprr-7kw8p7-frz', 'x-amz-id-2': 'mhkchprr-7kw8p7-frz', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchprr-7kw8p7-frz', 'HostId': 'mhkchprr-7kw8p7-frz', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchprr-7kw8p7-frz', 'x-amz-id-2': 'mhkchprr-7kw8p7-frz', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,292 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,293 [75] [DEBUG] [data.model.storage] Removing sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e from local_us namespacegcworker stdout | 2025-11-04 09:06:55,293 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,293 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e'} namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,294 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | a6d910222b0b21da9ba5c37e6fa380f5281b682e5f0c7cfb49f9c6f1e9aee72e namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | af75a558106a5be7d47d3ff0add7f38e20d113b39a7d9b07c1c9d044383529d0 namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=af75a558106a5be7d47d3ff0add7f38e20d113b39a7d9b07c1c9d044383529d0', 'amz-sdk-invocation-id': b'e92afbf4-87cd-4db4-a5f1-560285e73317', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,295 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,298 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchps0-7q8fmg-10c6', 'x-amz-id-2': 'mhkchps0-7q8fmg-10c6', 'ETag': '"70d6dc6bca01a22f9e1c718312b59128-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:55 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5642', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchps0-7q8fmg-10c6', 'HostId': 'mhkchps0-7q8fmg-10c6', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchps0-7q8fmg-10c6', 'x-amz-id-2': 'mhkchps0-7q8fmg-10c6', 'etag': '"70d6dc6bca01a22f9e1c718312b59128-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:55 GMT', 'content-type': 'application/octet-stream', 'content-length': '5642', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 55, tzinfo=tzutc()), 'ContentLength': 5642, 'ETag': '"70d6dc6bca01a22f9e1c718312b59128-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e'} namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,299 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,300 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1c2deedf181a34ed529e2dc35ab818d25a75bec633d4530fad45e6a64e82d236 namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 11ffb0aa49ab83139a0bb47e3a54071d02ea24d5cdb3a85f677c05a252ec2a49 namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,301 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=11ffb0aa49ab83139a0bb47e3a54071d02ea24d5cdb3a85f677c05a252ec2a49', 'amz-sdk-invocation-id': b'7a75816a-5210-48eb-968a-2c095de470ef', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,302 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/4d/4d369543d64802619e5ea6a3217702d9474e7e51ce4172c5248b171899be826e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchps6-7u40zs-1cra', 'x-amz-id-2': 'mhkchps6-7u40zs-1cra', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,307 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchps6-7u40zs-1cra', 'HostId': 'mhkchps6-7u40zs-1cra', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchps6-7u40zs-1cra', 'x-amz-id-2': 'mhkchps6-7u40zs-1cra', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,308 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,308 [75] [DEBUG] [data.model.storage] Removing sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e from local_us namespacegcworker stdout | 2025-11-04 09:06:55,308 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e'} namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,309 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | bc49badcb330537a08ecb7f736235b66782d4f37316d50be20a7276af3f89bc7 namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | a5398397d1618c52aa7498ff3d08e00821723a2f87e10bb64cc409b932922222 namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,310 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a5398397d1618c52aa7498ff3d08e00821723a2f87e10bb64cc409b932922222', 'amz-sdk-invocation-id': b'b0685a41-f67b-4e88-85a6-3a0f94ee5e4d', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,311 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsf-7zgq4z-sxe', 'x-amz-id-2': 'mhkchpsf-7zgq4z-sxe', 'ETag': '"3ffaf93ef0ab02979db522559590578f-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:01 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '79039', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpsf-7zgq4z-sxe', 'HostId': 'mhkchpsf-7zgq4z-sxe', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsf-7zgq4z-sxe', 'x-amz-id-2': 'mhkchpsf-7zgq4z-sxe', 'etag': '"3ffaf93ef0ab02979db522559590578f-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:01 GMT', 'content-type': 'application/octet-stream', 'content-length': '79039', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 1, tzinfo=tzutc()), 'ContentLength': 79039, 'ETag': '"3ffaf93ef0ab02979db522559590578f-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e'} namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,314 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | ed15a2515a50d1341dc39cd13bdd8f663a3a7e9bb2a63daaab4b7fb044dc3e86 namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | ce8bda408ea8c1a057bdffbf9f57e5f792a49412f43dd00473d330183ef05471 namespacegcworker stdout | 2025-11-04 09:06:55,315 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,316 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,316 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,316 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ce8bda408ea8c1a057bdffbf9f57e5f792a49412f43dd00473d330183ef05471', 'amz-sdk-invocation-id': b'90bd7ebf-54dc-40b4-856d-adbd6206f197', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,316 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/64/642bd9a87afb8912e1cfc95bfbe82adceac1c1b2025f41880b6cb75b6c0c916e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsl-82twl8-pup', 'x-amz-id-2': 'mhkchpsl-82twl8-pup', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpsl-82twl8-pup', 'HostId': 'mhkchpsl-82twl8-pup', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsl-82twl8-pup', 'x-amz-id-2': 'mhkchpsl-82twl8-pup', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,323 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,324 [75] [DEBUG] [data.model.storage] Removing sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,324 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,325 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,325 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60'} namespacegcworker stdout | 2025-11-04 09:06:55,325 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,326 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 3565ec3efc48522eb55de4b9d23bb88dea2dde309b22c862b26faf2d867250e4 namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1c14017e0b0ae4bd4b5e8d04c0bdb7512f5943dff14649c93ffe1e6946d7b847 namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1c14017e0b0ae4bd4b5e8d04c0bdb7512f5943dff14649c93ffe1e6946d7b847', 'amz-sdk-invocation-id': b'c09d27ee-aa56-4365-90e0-c6e8df412fe4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,327 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,330 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,330 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsw-898qc1-1m7', 'x-amz-id-2': 'mhkchpsw-898qc1-1m7', 'ETag': '"7a9542cf6c451ff89e27ab6782cc63b2-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:01 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5717', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,330 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpsw-898qc1-1m7', 'HostId': 'mhkchpsw-898qc1-1m7', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpsw-898qc1-1m7', 'x-amz-id-2': 'mhkchpsw-898qc1-1m7', 'etag': '"7a9542cf6c451ff89e27ab6782cc63b2-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:01 GMT', 'content-type': 'application/octet-stream', 'content-length': '5717', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 1, tzinfo=tzutc()), 'ContentLength': 5717, 'ETag': '"7a9542cf6c451ff89e27ab6782cc63b2-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60'} namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,331 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 7f05be40d33c5f248e01fdb1ba9070463f3b218dd5e90bfdc1361b031b5d9402 namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 9ffeea450aac69e4bebd3db3d13303104219f053615d4903d19705da27d95881 namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,332 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,333 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,333 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ffeea450aac69e4bebd3db3d13303104219f053615d4903d19705da27d95881', 'amz-sdk-invocation-id': b'e5a7056d-fe64-4c3f-b116-bac7d6bc31f9', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,333 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/aa/aa3813f2d7da392c21ed6d5dd675704c4911f681566aac59635a9b18705e6c60 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpt2-8csxsx-1x1', 'x-amz-id-2': 'mhkchpt2-8csxsx-1x1', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpt2-8csxsx-1x1', 'HostId': 'mhkchpt2-8csxsx-1x1', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpt2-8csxsx-1x1', 'x-amz-id-2': 'mhkchpt2-8csxsx-1x1', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,339 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,340 [75] [DEBUG] [data.model.storage] Removing sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee from local_us namespacegcworker stdout | 2025-11-04 09:06:55,340 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,340 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee'} namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,341 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 226141ed590f91ebcb6a3ad272269dab8b4b2bcb27472ffe1136caf5e9418a66 namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 9f5875eefcbb00a5c6a62bf8f5b0f141a91ae5a76b5e5261db4ed89adef581c9 namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9f5875eefcbb00a5c6a62bf8f5b0f141a91ae5a76b5e5261db4ed89adef581c9', 'amz-sdk-invocation-id': b'5f58f97c-c426-44c3-a25a-b36a87ce66e8', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,342 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,345 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,345 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchptb-8i8mhn-2lj', 'x-amz-id-2': 'mhkchptb-8i8mhn-2lj', 'ETag': '"753b145ff6db4e3b358fe0e389c37737-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:25:59 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '584', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,345 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchptb-8i8mhn-2lj', 'HostId': 'mhkchptb-8i8mhn-2lj', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchptb-8i8mhn-2lj', 'x-amz-id-2': 'mhkchptb-8i8mhn-2lj', 'etag': '"753b145ff6db4e3b358fe0e389c37737-1"', 'last-modified': 'Tue, 04 Nov 2025 08:25:59 GMT', 'content-type': 'application/octet-stream', 'content-length': '584', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 25, 59, tzinfo=tzutc()), 'ContentLength': 584, 'ETag': '"753b145ff6db4e3b358fe0e389c37737-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee'} namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,346 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,347 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 3e48fbba44d96701838c3e67587b4112bc82241cc0e934bde91d5858d20232df namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | b2f7afbc27d3cdd15a73b2bd5b3864ccb6dfddaa1432cb6f8e0e40c82d2a86df namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=b2f7afbc27d3cdd15a73b2bd5b3864ccb6dfddaa1432cb6f8e0e40c82d2a86df', 'amz-sdk-invocation-id': b'f7703d80-a810-4f37-a1d5-06f37d4570fb', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,348 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/cc/cc40df68500fd27710307003a860cca71de4502bd43c23940216d572e121b1ee HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpth-8lym6a-dlv', 'x-amz-id-2': 'mhkchpth-8lym6a-dlv', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpth-8lym6a-dlv', 'HostId': 'mhkchpth-8lym6a-dlv', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpth-8lym6a-dlv', 'x-amz-id-2': 'mhkchpth-8lym6a-dlv', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,354 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:55,355 [75] [DEBUG] [data.model.gc] Found 9 uploaded blobs to GC under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,355 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 38 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,355 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [38]) namespacegcworker stdout | 2025-11-04 09:06:55,357 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 39 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,357 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:55,359 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 40 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,359 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:55,360 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 41 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,360 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:55,362 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 42 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,362 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:55,363 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 43 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,363 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:55,365 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 44 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,365 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:55,366 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 45 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,366 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:55,368 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 46 under repository 7 namespacegcworker stdout | 2025-11-04 09:06:55,368 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:55,369 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {39, 40, 41, 42, 43, 44, 45, 46, 47} namespacegcworker stdout | 2025-11-04 09:06:55,369 [75] [DEBUG] [data.model.storage] Garbage collecting storage 39 namespacegcworker stdout | 2025-11-04 09:06:55,370 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,370 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [39, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,371 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:55,373 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:55,373 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:55,373 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [39]) namespacegcworker stdout | 2025-11-04 09:06:55,374 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2']) namespacegcworker stdout | 2025-11-04 09:06:55,375 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2']) namespacegcworker stdout | 2025-11-04 09:06:55,376 [75] [DEBUG] [data.model.storage] Garbage collecting storage 40 namespacegcworker stdout | 2025-11-04 09:06:55,377 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,378 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [40, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,379 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:55,379 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:55,380 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:55,380 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [40]) namespacegcworker stdout | 2025-11-04 09:06:55,381 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168']) namespacegcworker stdout | 2025-11-04 09:06:55,382 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168']) namespacegcworker stdout | 2025-11-04 09:06:55,383 [75] [DEBUG] [data.model.storage] Garbage collecting storage 41 namespacegcworker stdout | 2025-11-04 09:06:55,384 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,384 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [41, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,385 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:55,386 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:55,386 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:55,386 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [41]) namespacegcworker stdout | 2025-11-04 09:06:55,387 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6']) namespacegcworker stdout | 2025-11-04 09:06:55,387 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6']) namespacegcworker stdout | 2025-11-04 09:06:55,389 [75] [DEBUG] [data.model.storage] Garbage collecting storage 42 namespacegcworker stdout | 2025-11-04 09:06:55,389 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,390 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [42, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,392 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:55,392 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:55,393 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:55,393 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [42]) namespacegcworker stdout | 2025-11-04 09:06:55,394 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d']) namespacegcworker stdout | 2025-11-04 09:06:55,394 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d']) namespacegcworker stdout | 2025-11-04 09:06:55,396 [75] [DEBUG] [data.model.storage] Garbage collecting storage 43 namespacegcworker stdout | 2025-11-04 09:06:55,396 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,397 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [43, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,398 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:55,398 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:55,399 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:55,399 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [43]) namespacegcworker stdout | 2025-11-04 09:06:55,399 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1']) namespacegcworker stdout | 2025-11-04 09:06:55,400 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1']) namespacegcworker stdout | 2025-11-04 09:06:55,401 [75] [DEBUG] [data.model.storage] Garbage collecting storage 44 namespacegcworker stdout | 2025-11-04 09:06:55,402 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,402 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [44, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,403 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:55,404 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:55,404 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:55,405 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [44]) namespacegcworker stdout | 2025-11-04 09:06:55,405 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655']) namespacegcworker stdout | 2025-11-04 09:06:55,405 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655']) namespacegcworker stdout | 2025-11-04 09:06:55,407 [75] [DEBUG] [data.model.storage] Garbage collecting storage 45 namespacegcworker stdout | 2025-11-04 09:06:55,407 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,408 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [45, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,409 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:55,409 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:55,410 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:55,410 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [45]) namespacegcworker stdout | 2025-11-04 09:06:55,411 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e']) namespacegcworker stdout | 2025-11-04 09:06:55,411 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e']) namespacegcworker stdout | 2025-11-04 09:06:55,413 [75] [DEBUG] [data.model.storage] Garbage collecting storage 46 namespacegcworker stdout | 2025-11-04 09:06:55,413 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,414 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [46, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,415 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:55,415 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:55,415 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:55,416 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [46]) namespacegcworker stdout | 2025-11-04 09:06:55,417 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5']) namespacegcworker stdout | 2025-11-04 09:06:55,417 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5']) namespacegcworker stdout | 2025-11-04 09:06:55,419 [75] [DEBUG] [data.model.storage] Garbage collecting storage 47 namespacegcworker stdout | 2025-11-04 09:06:55,419 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,420 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [47, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,420 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:55,421 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:55,421 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:55,422 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:55,423 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31']) namespacegcworker stdout | 2025-11-04 09:06:55,423 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31']) namespacegcworker stdout | 2025-11-04 09:06:55,426 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,426 [75] [DEBUG] [data.model.storage] Removing sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,426 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655'} namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,427 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,428 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4e0e2cd58465c5fff9b71c33820642014a03ca6d5b4de007107e8a03544b463b namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 7f31c956224166244e8c1055663a5123935129b36c9e65d0a9cb795d7c5eb40a namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7f31c956224166244e8c1055663a5123935129b36c9e65d0a9cb795d7c5eb40a', 'amz-sdk-invocation-id': b'a6d2c172-372c-4fbf-81df-7feb547da496', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,429 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpvq-9y2h2c-n7g', 'x-amz-id-2': 'mhkchpvq-9y2h2c-n7g', 'ETag': '"7478be6b88f1ffd40bbbd0969bfa45ef-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:10 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '580', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,433 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpvq-9y2h2c-n7g', 'HostId': 'mhkchpvq-9y2h2c-n7g', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpvq-9y2h2c-n7g', 'x-amz-id-2': 'mhkchpvq-9y2h2c-n7g', 'etag': '"7478be6b88f1ffd40bbbd0969bfa45ef-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:10 GMT', 'content-type': 'application/octet-stream', 'content-length': '580', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 10, tzinfo=tzutc()), 'ContentLength': 580, 'ETag': '"7478be6b88f1ffd40bbbd0969bfa45ef-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655'} namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,434 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4706df00ae88849690133eb4ea03e639520cfab0ea6c751ee99cbdb1cf3ab11c namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 2e1ae353aca5d5ff74e529e69c17c0c4b08679e39aa29fe16fbad0c0c39e634d namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2e1ae353aca5d5ff74e529e69c17c0c4b08679e39aa29fe16fbad0c0c39e634d', 'amz-sdk-invocation-id': b'09b4b76a-426a-4c67-b2a6-8f206ee2d86b', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,435 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,441 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/d7/d70fe15f9be3d15fea659cd54e3267b83bc5eca667fd46f7ce256a9c5c76b655 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,441 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpvw-a1vee1-13wk', 'x-amz-id-2': 'mhkchpvw-a1vee1-13wk', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpvw-a1vee1-13wk', 'HostId': 'mhkchpvw-a1vee1-13wk', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpvw-a1vee1-13wk', 'x-amz-id-2': 'mhkchpvw-a1vee1-13wk', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,442 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,443 [75] [DEBUG] [data.model.storage] Removing sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,443 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31'} namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,444 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 9af545e9f8c01741c17187ab2d6d18542f7eac8a8ed103307860a17b6625c22a namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | ba64f98ece797e420061e08ffd34e1e9406149e0b466d7ec4c83fc7f3db53ddb namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ba64f98ece797e420061e08ffd34e1e9406149e0b466d7ec4c83fc7f3db53ddb', 'amz-sdk-invocation-id': b'7dd3ae21-60ef-4093-8b49-bfef31f61193', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,445 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,448 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpw6-a7nhnd-sag', 'x-amz-id-2': 'mhkchpw6-a7nhnd-sag', 'ETag': '"53322718ebdc1748c36dd9ac59b3279e-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:12 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpw6-a7nhnd-sag', 'HostId': 'mhkchpw6-a7nhnd-sag', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpw6-a7nhnd-sag', 'x-amz-id-2': 'mhkchpw6-a7nhnd-sag', 'etag': '"53322718ebdc1748c36dd9ac59b3279e-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:12 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 12, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"53322718ebdc1748c36dd9ac59b3279e-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31'} namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,449 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6880bd4f174d8fa2e566784b739188297d7f7336b0cbe5b89fdfe9df650be6e7 namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | d03577851d1abf7b19031ecb8757a5d4f40015e6c1aa334aaef30d765099d697 namespacegcworker stdout | 2025-11-04 09:06:55,450 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,451 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,451 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,451 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d03577851d1abf7b19031ecb8757a5d4f40015e6c1aa334aaef30d765099d697', 'amz-sdk-invocation-id': b'9293041a-062e-4814-8e75-4b43d98ba5c7', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,451 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/db/dbeeaa840de093e1b95605ea87f1b4de22ec102327070084d581ec7bc1364c31 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwc-ab00iw-14hb', 'x-amz-id-2': 'mhkchpwc-ab00iw-14hb', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,457 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpwc-ab00iw-14hb', 'HostId': 'mhkchpwc-ab00iw-14hb', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwc-ab00iw-14hb', 'x-amz-id-2': 'mhkchpwc-ab00iw-14hb', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,458 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,459 [75] [DEBUG] [data.model.storage] Removing sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,459 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168'} namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,460 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 451948734962334cb456ca4a88b765a805bfb9399be0ee901360f4fb1d0229fd namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 0fabaaffb4f8c5249a5df71aa54da04c649b8921832a054c433a91d78a169252 namespacegcworker stdout | 2025-11-04 09:06:55,461 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,462 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,462 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,462 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0fabaaffb4f8c5249a5df71aa54da04c649b8921832a054c433a91d78a169252', 'amz-sdk-invocation-id': b'e32bc1ec-ca05-49ea-b7ae-c03872166ce0', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,462 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwn-ahju4z-11ip', 'x-amz-id-2': 'mhkchpwn-ahju4z-11ip', 'ETag': '"d50e2d0022ada080b3855c780826a429-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:06 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5718', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpwn-ahju4z-11ip', 'HostId': 'mhkchpwn-ahju4z-11ip', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwn-ahju4z-11ip', 'x-amz-id-2': 'mhkchpwn-ahju4z-11ip', 'etag': '"d50e2d0022ada080b3855c780826a429-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:06 GMT', 'content-type': 'application/octet-stream', 'content-length': '5718', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 6, tzinfo=tzutc()), 'ContentLength': 5718, 'ETag': '"d50e2d0022ada080b3855c780826a429-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168'} namespacegcworker stdout | 2025-11-04 09:06:55,466 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 99e3627e0cab2ba9bca39af610667b7f969e07f5dfc0c885c0bda8b8b7265051 namespacegcworker stdout | 2025-11-04 09:06:55,467 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 926dbe34e3eed0b537463c0f491a57b14aaa5215fcb570e2a81d71258ae5306a namespacegcworker stdout | 2025-11-04 09:06:55,468 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,468 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,468 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,468 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=926dbe34e3eed0b537463c0f491a57b14aaa5215fcb570e2a81d71258ae5306a', 'amz-sdk-invocation-id': b'ec27fcac-b543-440c-8716-e483adea303d', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,468 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7e0e8d6ce05cff88c5a5220b9b7fab1e03122e76c49e794609354d1f7a9a168 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwt-al5h53-1ctj', 'x-amz-id-2': 'mhkchpwt-al5h53-1ctj', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpwt-al5h53-1ctj', 'HostId': 'mhkchpwt-al5h53-1ctj', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpwt-al5h53-1ctj', 'x-amz-id-2': 'mhkchpwt-al5h53-1ctj', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,475 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,476 [75] [DEBUG] [data.model.storage] Removing sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e from local_us namespacegcworker stdout | 2025-11-04 09:06:55,476 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e'} namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,477 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,478 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0eb6f4468feaf881d06803c24ba47cc3ca7cb21ecf03743f8c2ebe65e13c1562 namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 04578906944fb53a009ba0848da70a1e08f257483d1c2c7f5482355b27a06cb2 namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=04578906944fb53a009ba0848da70a1e08f257483d1c2c7f5482355b27a06cb2', 'amz-sdk-invocation-id': b'e7c96128-951c-49cb-8953-4516c6547cd4', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,479 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpx4-artya7-zwa', 'x-amz-id-2': 'mhkchpx4-artya7-zwa', 'ETag': '"1e89c8aabad0ee9c0a82c35ddf9cf538-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:11 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '79005', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpx4-artya7-zwa', 'HostId': 'mhkchpx4-artya7-zwa', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpx4-artya7-zwa', 'x-amz-id-2': 'mhkchpx4-artya7-zwa', 'etag': '"1e89c8aabad0ee9c0a82c35ddf9cf538-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:11 GMT', 'content-type': 'application/octet-stream', 'content-length': '79005', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 11, tzinfo=tzutc()), 'ContentLength': 79005, 'ETag': '"1e89c8aabad0ee9c0a82c35ddf9cf538-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,483 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e'} namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,484 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4216eca135f3ce91a973d6fc8dbbabb68c04b47326ee84d70bf43e2b0093b0a8 namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | c256089e6c2ab8d311c67945743db0c10aa6cc33ff7dfcfa27f997a428bed103 namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c256089e6c2ab8d311c67945743db0c10aa6cc33ff7dfcfa27f997a428bed103', 'amz-sdk-invocation-id': b'7fc0fc1e-d254-4299-b2c3-6d470ff9a486', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,485 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/fd/fdec4331f7983ecc43538c024e5777c434aad8cb1ddf08084a05a44390d6943e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxa-avmivb-4mt', 'x-amz-id-2': 'mhkchpxa-avmivb-4mt', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpxa-avmivb-4mt', 'HostId': 'mhkchpxa-avmivb-4mt', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxa-avmivb-4mt', 'x-amz-id-2': 'mhkchpxa-avmivb-4mt', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,492 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,493 [75] [DEBUG] [data.model.storage] Removing sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,493 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2'} namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,495 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | df2af972627e513be7c28a3cc6236a7b4481f897f0ce2e400bf3670584e1107a namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | d9ca260b4b3953f5366733149c5533b98bffbf0c47b2444b44ab2993dc73d71c namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d9ca260b4b3953f5366733149c5533b98bffbf0c47b2444b44ab2993dc73d71c', 'amz-sdk-invocation-id': b'acc7d276-d6e3-48a2-8c0c-48ad50c42781', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,496 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,500 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,500 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxl-b1ziv9-35k', 'x-amz-id-2': 'mhkchpxl-b1ziv9-35k', 'ETag': '"d79d1533d445974a6fac572ac13d1f99-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:05 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '584', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,500 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpxl-b1ziv9-35k', 'HostId': 'mhkchpxl-b1ziv9-35k', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxl-b1ziv9-35k', 'x-amz-id-2': 'mhkchpxl-b1ziv9-35k', 'etag': '"d79d1533d445974a6fac572ac13d1f99-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:05 GMT', 'content-type': 'application/octet-stream', 'content-length': '584', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 5, tzinfo=tzutc()), 'ContentLength': 584, 'ETag': '"d79d1533d445974a6fac572ac13d1f99-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2'} namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,501 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 4ddd8957200da1db1620dcfdf1631be42433bc389234a5b0c4a54b05004e49c2 namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 92e3c75640f228a002720b142f21ab2d318c8633d2b4c2c1beaa90769f8e75a3 namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=92e3c75640f228a002720b142f21ab2d318c8633d2b4c2c1beaa90769f8e75a3', 'amz-sdk-invocation-id': b'fc12991d-600d-4da2-afea-9fd52851a1e1', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,502 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/31/3139e75da5d8de02f503007200b5d1b3f8770caefa8b6a4c79adcf94832b4cc2 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxr-b5nkju-1a1s', 'x-amz-id-2': 'mhkchpxr-b5nkju-1a1s', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,508 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpxr-b5nkju-1a1s', 'HostId': 'mhkchpxr-b5nkju-1a1s', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpxr-b5nkju-1a1s', 'x-amz-id-2': 'mhkchpxr-b5nkju-1a1s', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,509 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,509 [75] [DEBUG] [data.model.storage] Removing sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,509 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1'} namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,510 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e9c329d306532d89f38d2689dc8691debfe554e10a22846026f2411eb4ccba13 namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 990a4cbb75b5c2ce841cc5328e391264cb5c35368de657c630b7755d84b388db namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,511 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=990a4cbb75b5c2ce841cc5328e391264cb5c35368de657c630b7755d84b388db', 'amz-sdk-invocation-id': b'305e8f5d-18a4-4f54-9aa0-a7fc0ff37343', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,512 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpy1-bb7doz-n4y', 'x-amz-id-2': 'mhkchpy1-bb7doz-n4y', 'ETag': '"b9ef788a2e9180083bf495551a582f85-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:09 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3647529', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,515 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpy1-bb7doz-n4y', 'HostId': 'mhkchpy1-bb7doz-n4y', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpy1-bb7doz-n4y', 'x-amz-id-2': 'mhkchpy1-bb7doz-n4y', 'etag': '"b9ef788a2e9180083bf495551a582f85-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:09 GMT', 'content-type': 'application/octet-stream', 'content-length': '3647529', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 9, tzinfo=tzutc()), 'ContentLength': 3647529, 'ETag': '"b9ef788a2e9180083bf495551a582f85-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1'} namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,516 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1090a075ab2e36296b035d9507d69e495fc30784230b35e05a82bdf11e9f45b4 namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 65107691fa5e5ae15d683333e28eb8b2ea8075bb1e1a7f80e5769942ff2b905a namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=65107691fa5e5ae15d683333e28eb8b2ea8075bb1e1a7f80e5769942ff2b905a', 'amz-sdk-invocation-id': b'2fd1c8cf-57c1-4f7a-a209-29f600f6c5f4', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,517 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,523 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/47/47a70fdc8ac4c1273de626dc7710d3e19cfd5b9f3e10cfc4b14602bdfffbffe1 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpy6-begnfr-yw8', 'x-amz-id-2': 'mhkchpy6-begnfr-yw8', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpy6-begnfr-yw8', 'HostId': 'mhkchpy6-begnfr-yw8', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpy6-begnfr-yw8', 'x-amz-id-2': 'mhkchpy6-begnfr-yw8', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,524 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,525 [75] [DEBUG] [data.model.storage] Removing sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d from local_us namespacegcworker stdout | 2025-11-04 09:06:55,525 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d'} namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,526 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f8500349d59ac3df642da60d38b2ae5921b656738a51abd614889dbe4dd265d6 namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 4bf7f171a2347d9fce4bff09693bff0b5507d4e9192d64ed052fe68839636fee namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4bf7f171a2347d9fce4bff09693bff0b5507d4e9192d64ed052fe68839636fee', 'amz-sdk-invocation-id': b'3a9e8319-9d75-472c-a9d6-e8d41bfde29c', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,527 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,531 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,531 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpyg-bkjahf-3c3', 'x-amz-id-2': 'mhkchpyg-bkjahf-3c3', 'ETag': '"84640d82a0cfebe3cecb2a047dce8a32-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:07 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,531 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,531 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,531 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpyg-bkjahf-3c3', 'HostId': 'mhkchpyg-bkjahf-3c3', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpyg-bkjahf-3c3', 'x-amz-id-2': 'mhkchpyg-bkjahf-3c3', 'etag': '"84640d82a0cfebe3cecb2a047dce8a32-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:07 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 7, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"84640d82a0cfebe3cecb2a047dce8a32-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d'} namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,532 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | ce85220472815ca04de317bfe3699a86644e0c37744b6884d78420cfee6c943a namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 2238fc8853bbc2accf9180b19d414deb97166980cf708faa7e17ccec032427d3 namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2238fc8853bbc2accf9180b19d414deb97166980cf708faa7e17ccec032427d3', 'amz-sdk-invocation-id': b'e3c99b9f-dc70-4c42-9b0c-7c21a5514fe0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,533 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c2/c26023b0d841656913ab6cac3f47943f150dd7e9a307d1300fda95eb157c962d HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpym-bo10pu-7i5', 'x-amz-id-2': 'mhkchpym-bo10pu-7i5', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpym-bo10pu-7i5', 'HostId': 'mhkchpym-bo10pu-7i5', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpym-bo10pu-7i5', 'x-amz-id-2': 'mhkchpym-bo10pu-7i5', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,539 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,540 [75] [DEBUG] [data.model.storage] Removing sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,540 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,541 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,541 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6'} namespacegcworker stdout | 2025-11-04 09:06:55,541 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,541 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,542 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 66ecf67c7917ac4e94f8f6b881fe75eff93bf5be87dd827addfae18a33b6adbf namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 61d7f97131c614cbf75760f3e2140c38693c34018b45db4e2443ac7bc84f1d69 namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=61d7f97131c614cbf75760f3e2140c38693c34018b45db4e2443ac7bc84f1d69', 'amz-sdk-invocation-id': b'fa2591f0-a937-4d54-a88f-4435d4e4b50f', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,543 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,546 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,546 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpyw-btunj2-1bza', 'x-amz-id-2': 'mhkchpyw-btunj2-1bza', 'ETag': '"0fa0286f95d7876eca11a087e46ad626-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:06 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '79035', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,547 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,547 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpyw-btunj2-1bza', 'HostId': 'mhkchpyw-btunj2-1bza', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpyw-btunj2-1bza', 'x-amz-id-2': 'mhkchpyw-btunj2-1bza', 'etag': '"0fa0286f95d7876eca11a087e46ad626-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:06 GMT', 'content-type': 'application/octet-stream', 'content-length': '79035', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 6, tzinfo=tzutc()), 'ContentLength': 79035, 'ETag': '"0fa0286f95d7876eca11a087e46ad626-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6'} namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,548 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e68e5f873b7802b90454e5f782da02df7a5de48424d0023c1e51ca10ff895aaa namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 122dc1fadf474055b9ea675db8a9d42b6332dee8df99a6873e62392cc68100a7 namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=122dc1fadf474055b9ea675db8a9d42b6332dee8df99a6873e62392cc68100a7', 'amz-sdk-invocation-id': b'0307c4b8-7174-41c9-b79c-9d0ce0df3302', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,549 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem notificationworker stdout | 2025-11-04 09:06:55,551 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:55,556 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/67/67fddcb95396e53298f4ac7df9a3d3b39dd0e8e18e97231294f852ec23e112c6 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpz2-bxlylw-1849', 'x-amz-id-2': 'mhkchpz2-bxlylw-1849', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpz2-bxlylw-1849', 'HostId': 'mhkchpz2-bxlylw-1849', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpz2-bxlylw-1849', 'x-amz-id-2': 'mhkchpz2-bxlylw-1849', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,557 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 1]) namespacegcworker stdout | 2025-11-04 09:06:55,558 [75] [DEBUG] [data.model.storage] Removing sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5 from local_us namespacegcworker stdout | 2025-11-04 09:06:55,558 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5'} namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,559 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 22f244989cbb8fcfdd479228a5a71e6d2dd544ca505d37ad8333f3c504bb638b namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | f2455fd4af69dc899f96ec33b77ed537ea8a1a4899158ba75bad83b39a259208 namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f2455fd4af69dc899f96ec33b77ed537ea8a1a4899158ba75bad83b39a259208', 'amz-sdk-invocation-id': b'be3287dd-0d60-4f71-9b5a-36b04b18c0fd', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,560 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpzd-c49pod-ss1', 'x-amz-id-2': 'mhkchpzd-c49pod-ss1', 'ETag': '"4a5205b4e80b938c24d031baf5e8c3d0-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:11 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5678', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,564 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpzd-c49pod-ss1', 'HostId': 'mhkchpzd-c49pod-ss1', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpzd-c49pod-ss1', 'x-amz-id-2': 'mhkchpzd-c49pod-ss1', 'etag': '"4a5205b4e80b938c24d031baf5e8c3d0-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:11 GMT', 'content-type': 'application/octet-stream', 'content-length': '5678', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 11, tzinfo=tzutc()), 'ContentLength': 5678, 'ETag': '"4a5205b4e80b938c24d031baf5e8c3d0-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5'} namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,565 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,566 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:55,566 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090655Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:55,566 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090655Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | cf5170da13c6c6db281153be0ee4baa64cbfd6579c91224cac58e64a72360905 namespacegcworker stdout | 2025-11-04 09:06:55,566 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 8a9fd47b26320db31c151847c59e03938dcf8ad035d89e13b678f3008eda609f namespacegcworker stdout | 2025-11-04 09:06:55,566 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,567 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:55,567 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,567 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090655Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8a9fd47b26320db31c151847c59e03938dcf8ad035d89e13b678f3008eda609f', 'amz-sdk-invocation-id': b'0ebd5892-a48f-4c93-aa60-a8bed18a5277', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:55,567 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/46/46fe55b2f760b35f41708fb01116dae87029350fffeb1d00f7dd0520587195c5 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpzk-c83qyd-12zj', 'x-amz-id-2': 'mhkchpzk-c83qyd-12zj', 'Date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:55,573 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,574 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:55,574 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchpzk-c83qyd-12zj', 'HostId': 'mhkchpzk-c83qyd-12zj', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchpzk-c83qyd-12zj', 'x-amz-id-2': 'mhkchpzk-c83qyd-12zj', 'date': 'Tue, 04 Nov 2025 09:06:55 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:55,574 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:55,575 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [7, 10]) namespacegcworker stdout | 2025-11-04 09:06:55,575 [75] [DEBUG] [peewee] ('DELETE FROM "quotarepositorysize" WHERE ("quotarepositorysize"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,578 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "quotarepositorysize" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,579 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,579 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,580 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,581 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,582 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "apprtag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,583 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,584 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,585 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,586 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,586 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestsecuritystatus" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,587 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tagpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,588 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,588 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,589 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorypermission" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,590 [75] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repositorypermission" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."id" <= %s))', [7, 502]) namespacegcworker stdout | 2025-11-04 09:06:55,592 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ((("repositorypermission"."repository_id" = %s) AND ("repositorypermission"."id" >= %s)) AND ("repositorypermission"."id" <= %s))', [7, 2, 2]) namespacegcworker stdout | 2025-11-04 09:06:55,594 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorypermission" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,595 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorybuild" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,596 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorybuildtrigger" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,598 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,599 [75] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."id" <= %s))', [7, 502]) namespacegcworker stdout | 2025-11-04 09:06:55,599 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryactioncount" WHERE ((("repositoryactioncount"."repository_id" = %s) AND ("repositoryactioncount"."id" >= %s)) AND ("repositoryactioncount"."id" <= %s))', [7, 2, 2]) namespacegcworker stdout | 2025-11-04 09:06:55,601 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,602 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "star" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,603 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "accesstoken" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,604 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorynotification" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,605 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,606 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,607 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryauthorizedemail" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,608 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "tagpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,608 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifestpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,609 [75] [DEBUG] [peewee] ('DELETE FROM "deletedrepository" WHERE ("deletedrepository"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,610 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [7, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,612 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuild" WHERE ("repositorybuild"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,612 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,613 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,613 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE ("repomirrorconfig"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,614 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE ("manifestsecuritystatus"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,614 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE ("manifestlabel"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,615 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE ("manifestchild"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,615 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE ("manifestblob"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,616 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,616 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE ("tagpullstatistics"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,617 [75] [DEBUG] [peewee] ('DELETE FROM "star" WHERE ("star"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,617 [75] [DEBUG] [peewee] ('DELETE FROM "repositorysearchscore" WHERE ("repositorysearchscore"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,619 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ("repositorypermission"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,620 [75] [DEBUG] [peewee] ('DELETE FROM "repositorynotification" WHERE ("repositorynotification"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,620 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,621 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryauthorizedemail" WHERE ("repositoryauthorizedemail"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,621 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryactioncount" WHERE ("repositoryactioncount"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,622 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorrule" WHERE ("repomirrorrule"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,623 [75] [DEBUG] [peewee] ('DELETE FROM "quotarepositorysize" WHERE ("quotarepositorysize"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,624 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE ("manifestpullstatistics"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,624 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,625 [75] [DEBUG] [peewee] ('DELETE FROM "deletedrepository" WHERE ("deletedrepository"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,625 [75] [DEBUG] [peewee] ('DELETE FROM "blobupload" WHERE ("blobupload"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,625 [75] [DEBUG] [peewee] ('DELETE FROM "apprtag" WHERE ("apprtag"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,626 [75] [DEBUG] [peewee] ('DELETE FROM "accesstoken" WHERE ("accesstoken"."repository_id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,626 [75] [DEBUG] [peewee] ('DELETE FROM "repository" WHERE ("repository"."id" = %s)', [7]) namespacegcworker stdout | 2025-11-04 09:06:55,631 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."namespace_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,632 [75] [DEBUG] [peewee] ('DELETE FROM "autoprunetaskstatus" WHERE ("autoprunetaskstatus"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,633 [75] [DEBUG] [peewee] ('DELETE FROM "namespaceautoprunepolicy" WHERE ("namespaceautoprunepolicy"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,633 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."organization_id", "t1"."role_id", "t1"."description" FROM "team" AS "t1" WHERE ("t1"."organization_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,634 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."client_id", "t1"."secure_client_secret", "t1"."fully_migrated", "t1"."redirect_uri", "t1"."application_uri", "t1"."organization_id", "t1"."name", "t1"."description", "t1"."gravatar_email" FROM "oauthapplication" AS "t1" WHERE ("t1"."organization_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,636 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."service_id", "t1"."repository_id", "t1"."connected_user_id", "t1"."secure_auth_token", "t1"."secure_private_key", "t1"."fully_migrated", "t1"."config", "t1"."write_token_id", "t1"."pull_robot_id", "t1"."enabled", "t1"."disabled_reason_id", "t1"."disabled_datetime", "t1"."successive_failure_count", "t1"."successive_internal_error_count" FROM "repositorybuildtrigger" AS "t1" WHERE ("t1"."connected_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,636 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['95595bdd-7ef2-4730-9b0a-15a675d82fb0']) namespacegcworker stdout | 2025-11-04 09:06:55,638 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed", "t2"."id", "t2"."robot_account_id", "t2"."token", "t2"."fully_migrated", "t3"."id", "t3"."robot_account_id", "t3"."description", "t3"."unstructured_json" FROM "user" AS "t1" LEFT OUTER JOIN "robotaccountmetadata" AS "t3" ON ("t3"."robot_account_id" = "t1"."id") INNER JOIN "robotaccounttoken" AS "t2" ON ("t2"."robot_account_id" = "t1"."id") WHERE (("t1"."robot" = %s) AND ("t1"."username" ILIKE %s))', [True, '95595bdd-7ef2-4730-9b0a-15a675d82fb0+%']) namespacegcworker stdout | 2025-11-04 09:06:55,639 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE (0 = 1)', []) namespacegcworker stdout | 2025-11-04 09:06:55,639 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed", "t2"."id", "t2"."robot_account_id", "t2"."token", "t2"."fully_migrated", "t3"."id", "t3"."robot_account_id", "t3"."description", "t3"."unstructured_json" FROM "user" AS "t1" LEFT OUTER JOIN "robotaccountmetadata" AS "t3" ON ("t3"."robot_account_id" = "t1"."id") INNER JOIN "robotaccounttoken" AS "t2" ON ("t2"."robot_account_id" = "t1"."id") WHERE (("t1"."robot" = %s) AND ("t1"."username" ILIKE %s))', [True, '95595bdd-7ef2-4730-9b0a-15a675d82fb0+%']) namespacegcworker stdout | 2025-11-04 09:06:55,640 [75] [DEBUG] [peewee] ('UPDATE "servicekeyapproval" SET "approver_id" = %s WHERE ("servicekeyapproval"."approver_id" = %s)', [None, 3]) namespacegcworker stdout | 2025-11-04 09:06:55,642 [75] [DEBUG] [peewee] ('DELETE FROM "federatedlogin" WHERE ("federatedlogin"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,642 [75] [DEBUG] [peewee] ('DELETE FROM "quotanamespacesize" WHERE ("quotanamespacesize"."namespace_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,643 [75] [DEBUG] [peewee] ('DELETE FROM "oauthassignedtoken" WHERE ("oauthassignedtoken"."assigned_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,644 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuild" WHERE ("repositorybuild"."pull_robot_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,645 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."pull_robot_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,645 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."connected_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,646 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE ("repomirrorconfig"."internal_robot_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,646 [75] [DEBUG] [peewee] ('DELETE FROM "teammemberinvite" WHERE ("teammemberinvite"."inviter_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,647 [75] [DEBUG] [peewee] ('DELETE FROM "teammemberinvite" WHERE ("teammemberinvite"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,647 [75] [DEBUG] [peewee] ('DELETE FROM "teammember" WHERE ("teammember"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,648 [75] [DEBUG] [peewee] ('DELETE FROM "star" WHERE ("star"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,648 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ("repositorypermission"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,649 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,649 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."delegate_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,650 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."activating_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,650 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."org_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,651 [75] [DEBUG] [peewee] ('DELETE FROM "oauthassignedtoken" WHERE ("oauthassignedtoken"."assigned_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,651 [75] [DEBUG] [peewee] ('DELETE FROM "oauthaccesstoken" WHERE ("oauthaccesstoken"."authorized_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,652 [75] [DEBUG] [peewee] ('DELETE FROM "userregion" WHERE ("userregion"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,652 [75] [DEBUG] [peewee] ('DELETE FROM "userprompt" WHERE ("userprompt"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,653 [75] [DEBUG] [peewee] ('DELETE FROM "userorganizationquota" WHERE ("userorganizationquota"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,654 [75] [DEBUG] [peewee] ('DELETE FROM "team" WHERE ("team"."organization_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,654 [75] [DEBUG] [peewee] ('DELETE FROM "servicekeyapproval" WHERE ("servicekeyapproval"."approver_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,654 [75] [DEBUG] [peewee] ('DELETE FROM "robotaccounttoken" WHERE ("robotaccounttoken"."robot_account_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,655 [75] [DEBUG] [peewee] ('DELETE FROM "robotaccountmetadata" WHERE ("robotaccountmetadata"."robot_account_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,655 [75] [DEBUG] [peewee] ('DELETE FROM "repository" WHERE ("repository"."namespace_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,656 [75] [DEBUG] [peewee] ('DELETE FROM "redhatsubscriptions" WHERE ("redhatsubscriptions"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,657 [75] [DEBUG] [peewee] ('DELETE FROM "quotanamespacesize" WHERE ("quotanamespacesize"."namespace_user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,657 [75] [DEBUG] [peewee] ('DELETE FROM "proxycacheconfig" WHERE ("proxycacheconfig"."organization_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,658 [75] [DEBUG] [peewee] ('DELETE FROM "organizationrhskus" WHERE ("organizationrhskus"."org_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,659 [75] [DEBUG] [peewee] ('DELETE FROM "organizationrhskus" WHERE ("organizationrhskus"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,660 [75] [DEBUG] [peewee] ('DELETE FROM "oauthapplication" WHERE ("oauthapplication"."organization_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,660 [75] [DEBUG] [peewee] ('DELETE FROM "notification" WHERE ("notification"."target_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,661 [75] [DEBUG] [peewee] ('DELETE FROM "namespacegeorestriction" WHERE ("namespacegeorestriction"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,661 [75] [DEBUG] [peewee] ('DELETE FROM "namespaceautoprunepolicy" WHERE ("namespaceautoprunepolicy"."namespace_id" = %s)', [3]) gcworker stdout | 2025-11-04 09:06:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:06:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:07.663502+00:00 (in 12.001434 seconds) gcworker stdout | 2025-11-04 09:06:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:25 GMT)" (scheduled at 2025-11-04 09:06:55.661677+00:00) namespacegcworker stdout | 2025-11-04 09:06:55,662 [75] [DEBUG] [peewee] ('DELETE FROM "federatedlogin" WHERE ("federatedlogin"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,662 [75] [DEBUG] [peewee] ('DELETE FROM "emailconfirmation" WHERE ("emailconfirmation"."user_id" = %s)', [3]) gcworker stdout | 2025-11-04 09:06:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037615662, 1209600, True, 3, 500, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,663 [75] [DEBUG] [peewee] ('DELETE FROM "deletednamespace" WHERE ("deletednamespace"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,665 [75] [DEBUG] [peewee] ('DELETE FROM "autoprunetaskstatus" WHERE ("autoprunetaskstatus"."namespace_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,666 [75] [DEBUG] [peewee] ('DELETE FROM "appspecificauthtoken" WHERE ("appspecificauthtoken"."user_id" = %s)', [3]) namespacegcworker stdout | 2025-11-04 09:06:55,666 [75] [DEBUG] [peewee] ('DELETE FROM "user" WHERE ("user"."id" = %s)', [3]) gcworker stdout | 2025-11-04 09:06:55,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:06:55,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:06:55,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:25 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:06:55,673 [75] [DEBUG] [util.locking] Releasing lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:55,673 [75] [DEBUG] [redis_lock.release] Releasing 'lock:LARGE_GARBAGE_COLLECTION'. namespacegcworker stdout | 2025-11-04 09:06:55,674 [75] [DEBUG] [util.locking] Released lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:55,675 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."id" = %s)', [2]) namespacegcworker stdout | 2025-11-04 09:06:55,677 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 55, 677099), True, datetime.datetime(2025, 11, 4, 9, 6, 55, 677099), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,679 [75] [DEBUG] [peewee] ('UPDATE "queueitem" SET "available" = %s, "processing_expires" = %s, "retries_remaining" = ("queueitem"."retries_remaining" - %s), "state_id" = %s WHERE (("queueitem"."id" = %s) AND ("queueitem"."state_id" = %s))', [False, datetime.datetime(2025, 11, 4, 12, 6, 55, 677099), 1, '2a3284dd-3029-4a96-b015-7d27c973262a', 1, '1a57b541-615f-4839-9c24-d8c436737a5f']) namespacegcworker stdout | 2025-11-04 09:06:55,680 [75] [DEBUG] [workers.queueworker] Queue gave us some work: {"marker_id": 1, "original_username": "testorg2"} namespacegcworker stdout | 2025-11-04 09:06:55,680 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:06:55,680 [75] [DEBUG] [util.locking] Acquiring global lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:55,680 [75] [DEBUG] [redis_lock.acquire] Getting 'lock:LARGE_GARBAGE_COLLECTION' ... namespacegcworker stdout | 2025-11-04 09:06:55,683 [75] [INFO] [redis_lock.acquire] Got lock for 'lock:LARGE_GARBAGE_COLLECTION'. namespacegcworker stdout | 2025-11-04 09:06:55,683 [75] [DEBUG] [util.locking] Acquired lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:55,683 [75] [DEBUG] [__main__] Got namespace GC queue item: {'marker_id': 1, 'original_username': 'testorg2'} namespacegcworker stdout | 2025-11-04 09:06:55,684 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."marked", "t1"."original_username", "t1"."original_email", "t1"."queue_id" FROM "deletednamespace" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,689 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [5, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,690 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['dockerfilebuild/04617bc0-5074-4229-9551-bb696f9ce5fe/%']) namespacegcworker stdout | 2025-11-04 09:06:55,691 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['notification/04617bc0-5074-4229-9551-bb696f9ce5fe/%']) namespacegcworker stdout | 2025-11-04 09:06:55,692 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."queue_name" ILIKE %s)', ['repositorygc/04617bc0-5074-4229-9551-bb696f9ce5fe/%']) namespacegcworker stdout | 2025-11-04 09:06:55,693 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."namespace_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:55,694 [75] [DEBUG] [peewee] ('UPDATE "repository" SET "namespace_user_id" = %s, "name" = %s, "visibility_id" = %s, "description" = %s, "badge_token" = %s, "kind_id" = %s, "trust_enabled" = %s, "state" = %s WHERE ("repository"."id" = %s)', [5, 'alpine', 2, None, '2e04a59f-a1b0-4f52-8be5-b1cbccd05e05', 1, False, 3, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,697 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,698 [75] [DEBUG] [data.model.gc] Purging repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,698 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:55,700 [75] [DEBUG] [data.model.gc] Found 10 tags to GC under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,700 [75] [DEBUG] [data.model.gc] Deleting tag 19 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,700 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,701 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-bd4bd90c-ec12-4f29-a047-975c9899baa4']) namespacegcworker stdout | 2025-11-04 09:06:55,702 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,703 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-bd4bd90c-ec12-4f29-a047-975c9899baa4 namespacegcworker stdout | 2025-11-04 09:06:55,703 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,705 [75] [DEBUG] [data.model.gc] Deleting tag 20 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,705 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,706 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-4307e456-766c-4a3a-83e9-b458b707b4fd']) namespacegcworker stdout | 2025-11-04 09:06:55,707 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,707 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-4307e456-766c-4a3a-83e9-b458b707b4fd namespacegcworker stdout | 2025-11-04 09:06:55,707 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,709 [75] [DEBUG] [data.model.gc] Deleting tag 21 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,709 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,710 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-c85507fc-2c23-4986-8c13-bf873bb1054f']) namespacegcworker stdout | 2025-11-04 09:06:55,710 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,711 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-c85507fc-2c23-4986-8c13-bf873bb1054f namespacegcworker stdout | 2025-11-04 09:06:55,711 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,712 [75] [DEBUG] [data.model.gc] Deleting tag 22 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,713 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,714 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-42e53c15-8204-432e-ba7a-91f84c3cd64c']) namespacegcworker stdout | 2025-11-04 09:06:55,714 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,714 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-42e53c15-8204-432e-ba7a-91f84c3cd64c namespacegcworker stdout | 2025-11-04 09:06:55,715 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,716 [75] [DEBUG] [data.model.gc] Deleting tag 23 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,717 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,717 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-ba864ead-ed24-4b4e-9a75-dfb336220667']) namespacegcworker stdout | 2025-11-04 09:06:55,718 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,719 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-ba864ead-ed24-4b4e-9a75-dfb336220667 namespacegcworker stdout | 2025-11-04 09:06:55,720 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,721 [75] [DEBUG] [data.model.gc] Deleting tag 24 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,722 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,722 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-b6fc482a-e2b1-4b02-a868-b9611eb9766c']) namespacegcworker stdout | 2025-11-04 09:06:55,722 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,723 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-b6fc482a-e2b1-4b02-a868-b9611eb9766c namespacegcworker stdout | 2025-11-04 09:06:55,723 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,724 [75] [DEBUG] [data.model.gc] Deleting tag 25 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,725 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,726 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-8bbcfad0-2e28-495f-8870-2da0f438c1d7']) namespacegcworker stdout | 2025-11-04 09:06:55,726 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,726 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-8bbcfad0-2e28-495f-8870-2da0f438c1d7 namespacegcworker stdout | 2025-11-04 09:06:55,726 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,728 [75] [DEBUG] [data.model.gc] Deleting tag 26 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,728 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,729 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-cf78582f-9975-4079-91c3-051ea7ed6439']) namespacegcworker stdout | 2025-11-04 09:06:55,729 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,729 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-cf78582f-9975-4079-91c3-051ea7ed6439 namespacegcworker stdout | 2025-11-04 09:06:55,729 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,731 [75] [DEBUG] [data.model.gc] Deleting tag 27 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,731 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,732 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-144f44d1-baec-4548-8447-e852fd5250cb']) namespacegcworker stdout | 2025-11-04 09:06:55,732 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,732 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-144f44d1-baec-4548-8447-e852fd5250cb namespacegcworker stdout | 2025-11-04 09:06:55,732 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,734 [75] [DEBUG] [data.model.gc] Deleting tag 28 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:55,734 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,735 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-4f1c0600-2a11-4693-95f4-4ed79c770b09']) namespacegcworker stdout | 2025-11-04 09:06:55,735 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,735 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-4f1c0600-2a11-4693-95f4-4ed79c770b09 namespacegcworker stdout | 2025-11-04 09:06:55,736 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,737 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,738 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,739 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,741 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,742 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,742 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,745 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,745 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,746 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,747 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [19, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,748 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [19, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,748 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [19, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,749 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [19, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,751 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [19, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,751 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,752 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,752 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,752 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [19, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,753 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:85f2b723e106c34644cd5851d7e81ee87da98ac54672b29947c052a45d31dc2f']) namespacegcworker stdout | 2025-11-04 09:06:55,754 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [19]) namespacegcworker stdout | 2025-11-04 09:06:55,756 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:85f2b723e106c34644cd5851d7e81ee87da98ac54672b29947c052a45d31dc2f', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,757 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,757 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:85f2b723e106c34644cd5851d7e81ee87da98ac54672b29947c052a45d31dc2f namespacegcworker stdout | 2025-11-04 09:06:55,769 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:85f2b723e106c34644cd5851d7e81ee87da98ac54672b29947c052a45d31dc2f HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,770 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:85f2b723e106c34644cd5851d7e81ee87da98ac54672b29947c052a45d31dc2f from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,770 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,771 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,771 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,773 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,773 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,774 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,774 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,775 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,776 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,777 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [20, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,778 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [20, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,779 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [20, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,779 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [20, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,780 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [20, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,780 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,780 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,781 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,781 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [20, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,781 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:0de8d0f587c8bf25bf1381e379cdccf985e878ac949743f45365b4872344ba17']) namespacegcworker stdout | 2025-11-04 09:06:55,782 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [20]) namespacegcworker stdout | 2025-11-04 09:06:55,784 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:0de8d0f587c8bf25bf1381e379cdccf985e878ac949743f45365b4872344ba17', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,785 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,785 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:0de8d0f587c8bf25bf1381e379cdccf985e878ac949743f45365b4872344ba17 namespacegcworker stdout | 2025-11-04 09:06:55,791 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:0de8d0f587c8bf25bf1381e379cdccf985e878ac949743f45365b4872344ba17 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,791 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:0de8d0f587c8bf25bf1381e379cdccf985e878ac949743f45365b4872344ba17 from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,792 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,793 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,794 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,795 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,795 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,796 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,797 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,797 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,798 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,799 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [21, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,800 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [21, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,801 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [21, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,801 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [21, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,803 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [21, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,803 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,804 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,804 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,804 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [21, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,804 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:0a23e5e40e97dbde2e957d31dce1e97135d77639f92600cd8176e30dea70c24f']) namespacegcworker stdout | 2025-11-04 09:06:55,805 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [21]) namespacegcworker stdout | 2025-11-04 09:06:55,807 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:0a23e5e40e97dbde2e957d31dce1e97135d77639f92600cd8176e30dea70c24f', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,808 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,808 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:0a23e5e40e97dbde2e957d31dce1e97135d77639f92600cd8176e30dea70c24f namespacegcworker stdout | 2025-11-04 09:06:55,818 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:0a23e5e40e97dbde2e957d31dce1e97135d77639f92600cd8176e30dea70c24f HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,819 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:0a23e5e40e97dbde2e957d31dce1e97135d77639f92600cd8176e30dea70c24f from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,819 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,820 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,821 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,822 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,823 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,825 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,825 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,826 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,827 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,827 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [22, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,828 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [22, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,828 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [22, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,829 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [22, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,829 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [22, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,830 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,830 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,831 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,831 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [22, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,831 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:de2c58cc2a1ecdc061938bd6efbfb70b1b749c247e89a055dd92375120e3ba3d']) namespacegcworker stdout | 2025-11-04 09:06:55,832 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [22]) namespacegcworker stdout | 2025-11-04 09:06:55,834 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:de2c58cc2a1ecdc061938bd6efbfb70b1b749c247e89a055dd92375120e3ba3d', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,835 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,835 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:de2c58cc2a1ecdc061938bd6efbfb70b1b749c247e89a055dd92375120e3ba3d namespacegcworker stdout | 2025-11-04 09:06:55,845 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:de2c58cc2a1ecdc061938bd6efbfb70b1b749c247e89a055dd92375120e3ba3d HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,845 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:de2c58cc2a1ecdc061938bd6efbfb70b1b749c247e89a055dd92375120e3ba3d from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,846 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,846 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,848 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,850 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,851 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,851 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,852 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,853 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,854 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,854 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [23, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,856 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [23, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,856 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [23, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,857 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [23, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,857 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [23, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,858 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,858 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,858 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,858 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [23, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,859 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:f6b2f75fca5a2c6c2cbb27cc286e778f1c0ba732e46ed33a440d7ce9926c1c29']) namespacegcworker stdout | 2025-11-04 09:06:55,859 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [23]) namespacegcworker stdout | 2025-11-04 09:06:55,861 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:f6b2f75fca5a2c6c2cbb27cc286e778f1c0ba732e46ed33a440d7ce9926c1c29', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,862 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,862 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:f6b2f75fca5a2c6c2cbb27cc286e778f1c0ba732e46ed33a440d7ce9926c1c29 namespacegcworker stdout | 2025-11-04 09:06:55,873 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:f6b2f75fca5a2c6c2cbb27cc286e778f1c0ba732e46ed33a440d7ce9926c1c29 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,874 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:f6b2f75fca5a2c6c2cbb27cc286e778f1c0ba732e46ed33a440d7ce9926c1c29 from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,875 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,876 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,876 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,877 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,878 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,879 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,879 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,880 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,881 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,881 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [24, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,882 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [24, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,883 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [24, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,883 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [24, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,884 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [24, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,884 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,884 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,885 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,885 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [24, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,885 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:edb0999a146533529ece3bc051fd424d3cb7c395f5a6e8a27442fcd0509ed73d']) namespacegcworker stdout | 2025-11-04 09:06:55,885 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [24]) namespacegcworker stdout | 2025-11-04 09:06:55,888 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:edb0999a146533529ece3bc051fd424d3cb7c395f5a6e8a27442fcd0509ed73d', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,889 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,889 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:edb0999a146533529ece3bc051fd424d3cb7c395f5a6e8a27442fcd0509ed73d namespacegcworker stdout | 2025-11-04 09:06:55,894 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:edb0999a146533529ece3bc051fd424d3cb7c395f5a6e8a27442fcd0509ed73d HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,895 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:edb0999a146533529ece3bc051fd424d3cb7c395f5a6e8a27442fcd0509ed73d from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,895 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,896 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,897 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,898 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,898 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,899 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,900 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,900 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,901 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,902 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [25, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,903 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [25, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,903 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [25, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,904 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [25, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,904 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [25, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,905 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,905 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,905 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,906 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [25, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,906 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:24bb3511a0db7b5114a4aee033c65a8a4148f39b7b80a398e548546db967a36f']) namespacegcworker stdout | 2025-11-04 09:06:55,907 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [25]) namespacegcworker stdout | 2025-11-04 09:06:55,909 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:24bb3511a0db7b5114a4aee033c65a8a4148f39b7b80a398e548546db967a36f', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,910 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,910 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:24bb3511a0db7b5114a4aee033c65a8a4148f39b7b80a398e548546db967a36f namespacegcworker stdout | 2025-11-04 09:06:55,920 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:24bb3511a0db7b5114a4aee033c65a8a4148f39b7b80a398e548546db967a36f HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,921 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:24bb3511a0db7b5114a4aee033c65a8a4148f39b7b80a398e548546db967a36f from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,921 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,923 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,923 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,925 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,925 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,926 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,926 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,927 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,928 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,929 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [26, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,930 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [26, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,930 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [26, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,930 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [26, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,931 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [26, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,932 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,932 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,932 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,933 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [26, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,933 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:610eaa71e733bffea8d63705b48f33eb0c992d5304c608a5dcfffcc9c3cbcf2e']) namespacegcworker stdout | 2025-11-04 09:06:55,933 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [26]) namespacegcworker stdout | 2025-11-04 09:06:55,935 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:610eaa71e733bffea8d63705b48f33eb0c992d5304c608a5dcfffcc9c3cbcf2e', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,936 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,936 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:610eaa71e733bffea8d63705b48f33eb0c992d5304c608a5dcfffcc9c3cbcf2e buildlogsarchiver stdout | 2025-11-04 09:06:55,938 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:06:55,941 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:610eaa71e733bffea8d63705b48f33eb0c992d5304c608a5dcfffcc9c3cbcf2e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,941 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:610eaa71e733bffea8d63705b48f33eb0c992d5304c608a5dcfffcc9c3cbcf2e from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,942 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,943 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,943 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,944 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,945 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,946 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,947 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,948 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,949 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,949 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [27, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,950 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [27, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,951 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [27, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,951 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [27, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,952 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [27, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,952 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,953 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,953 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,954 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [27, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,954 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:9eec16c5eada75150a82666ba0ad6df76b164a6f8582ba5cb964c0813fa56625']) namespacegcworker stdout | 2025-11-04 09:06:55,954 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [27]) namespacegcworker stdout | 2025-11-04 09:06:55,956 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:9eec16c5eada75150a82666ba0ad6df76b164a6f8582ba5cb964c0813fa56625', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,957 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,957 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:9eec16c5eada75150a82666ba0ad6df76b164a6f8582ba5cb964c0813fa56625 namespacegcworker stdout | 2025-11-04 09:06:55,968 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:9eec16c5eada75150a82666ba0ad6df76b164a6f8582ba5cb964c0813fa56625 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,968 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:9eec16c5eada75150a82666ba0ad6df76b164a6f8582ba5cb964c0813fa56625 from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,969 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,970 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,971 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,972 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,973 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,973 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,974 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,975 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,975 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,976 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [28, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,977 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [28, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,978 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [28, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,979 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [28, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,979 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [28, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,980 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,982 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:55,982 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:55,982 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [28, 8]) namespacegcworker stdout | 2025-11-04 09:06:55,983 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:f7c453b4ef3c0889d67578b1a285304ec5728078638c55cb64bcca996c7ae2b1']) namespacegcworker stdout | 2025-11-04 09:06:55,983 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [28]) namespacegcworker stdout | 2025-11-04 09:06:55,985 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:f7c453b4ef3c0889d67578b1a285304ec5728078638c55cb64bcca996c7ae2b1', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,986 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:55,986 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:f7c453b4ef3c0889d67578b1a285304ec5728078638c55cb64bcca996c7ae2b1 namespacegcworker stdout | 2025-11-04 09:06:55,991 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:f7c453b4ef3c0889d67578b1a285304ec5728078638c55cb64bcca996c7ae2b1 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:55,991 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:f7c453b4ef3c0889d67578b1a285304ec5728078638c55cb64bcca996c7ae2b1 from secscan service namespacegcworker stdout | 2025-11-04 09:06:55,991 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,992 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:55,994 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,995 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:55,996 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,997 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:55,999 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:55,999 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,001 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,001 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,003 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,003 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,005 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,005 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,007 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,008 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,009 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,010 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,011 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,012 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,013 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,014 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,015 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,016 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,018 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,018 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,020 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,021 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,022 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,023 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,024 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,025 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,026 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,027 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,029 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,030 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,032 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,032 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,034 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,034 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [76]) namespacegcworker stdout | 2025-11-04 09:06:56,036 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [77, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,037 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [77]) namespacegcworker stdout | 2025-11-04 09:06:56,038 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [78, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,039 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [78]) namespacegcworker stdout | 2025-11-04 09:06:56,040 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [79, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,041 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [79]) namespacegcworker stdout | 2025-11-04 09:06:56,042 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [80, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,043 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [80]) namespacegcworker stdout | 2025-11-04 09:06:56,045 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [81, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,045 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [81]) namespacegcworker stdout | 2025-11-04 09:06:56,047 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [82, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,048 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [82]) namespacegcworker stdout | 2025-11-04 09:06:56,050 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [83, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,051 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [83]) namespacegcworker stdout | 2025-11-04 09:06:56,052 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [84, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,053 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [84]) namespacegcworker stdout | 2025-11-04 09:06:56,054 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [85, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,055 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [85]) namespacegcworker stdout | 2025-11-04 09:06:56,057 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [86, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,057 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [86]) namespacegcworker stdout | 2025-11-04 09:06:56,059 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [87, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,060 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [87]) namespacegcworker stdout | 2025-11-04 09:06:56,061 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [88, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,062 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [88]) namespacegcworker stdout | 2025-11-04 09:06:56,063 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [89, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,064 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [89]) namespacegcworker stdout | 2025-11-04 09:06:56,066 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [90, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,066 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [90]) namespacegcworker stdout | 2025-11-04 09:06:56,068 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [91, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,069 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [91]) namespacegcworker stdout | 2025-11-04 09:06:56,071 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71} namespacegcworker stdout | 2025-11-04 09:06:56,071 [75] [DEBUG] [data.model.storage] Garbage collecting storage 48 namespacegcworker stdout | 2025-11-04 09:06:56,071 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,072 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,073 [75] [DEBUG] [data.model.storage] Garbage collecting storage 49 namespacegcworker stdout | 2025-11-04 09:06:56,073 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,074 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,074 [75] [DEBUG] [data.model.storage] Garbage collecting storage 50 namespacegcworker stdout | 2025-11-04 09:06:56,074 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,075 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,076 [75] [DEBUG] [data.model.storage] Garbage collecting storage 51 namespacegcworker stdout | 2025-11-04 09:06:56,076 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,077 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,078 [75] [DEBUG] [data.model.storage] Garbage collecting storage 52 namespacegcworker stdout | 2025-11-04 09:06:56,079 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,079 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,080 [75] [DEBUG] [data.model.storage] Garbage collecting storage 53 namespacegcworker stdout | 2025-11-04 09:06:56,080 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,081 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,082 [75] [DEBUG] [data.model.storage] Garbage collecting storage 54 namespacegcworker stdout | 2025-11-04 09:06:56,082 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,082 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,083 [75] [DEBUG] [data.model.storage] Garbage collecting storage 55 namespacegcworker stdout | 2025-11-04 09:06:56,083 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,084 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,085 [75] [DEBUG] [data.model.storage] Garbage collecting storage 56 namespacegcworker stdout | 2025-11-04 09:06:56,085 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,086 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,086 [75] [DEBUG] [data.model.storage] Garbage collecting storage 57 namespacegcworker stdout | 2025-11-04 09:06:56,087 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,087 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,088 [75] [DEBUG] [data.model.storage] Garbage collecting storage 58 namespacegcworker stdout | 2025-11-04 09:06:56,088 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,089 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,089 [75] [DEBUG] [data.model.storage] Garbage collecting storage 59 namespacegcworker stdout | 2025-11-04 09:06:56,090 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,090 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,091 [75] [DEBUG] [data.model.storage] Garbage collecting storage 60 namespacegcworker stdout | 2025-11-04 09:06:56,091 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,092 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,092 [75] [DEBUG] [data.model.storage] Garbage collecting storage 61 namespacegcworker stdout | 2025-11-04 09:06:56,092 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,093 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,094 [75] [DEBUG] [data.model.storage] Garbage collecting storage 62 namespacegcworker stdout | 2025-11-04 09:06:56,094 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,095 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,096 [75] [DEBUG] [data.model.storage] Garbage collecting storage 63 namespacegcworker stdout | 2025-11-04 09:06:56,096 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,097 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,098 [75] [DEBUG] [data.model.storage] Garbage collecting storage 64 namespacegcworker stdout | 2025-11-04 09:06:56,098 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,099 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,099 [75] [DEBUG] [data.model.storage] Garbage collecting storage 65 namespacegcworker stdout | 2025-11-04 09:06:56,099 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,101 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,101 [75] [DEBUG] [data.model.storage] Garbage collecting storage 66 namespacegcworker stdout | 2025-11-04 09:06:56,102 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,103 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,103 [75] [DEBUG] [data.model.storage] Garbage collecting storage 67 namespacegcworker stdout | 2025-11-04 09:06:56,104 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,105 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,105 [75] [DEBUG] [data.model.storage] Garbage collecting storage 68 namespacegcworker stdout | 2025-11-04 09:06:56,105 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,106 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,107 [75] [DEBUG] [data.model.storage] Garbage collecting storage 69 namespacegcworker stdout | 2025-11-04 09:06:56,107 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,108 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,109 [75] [DEBUG] [data.model.storage] Garbage collecting storage 70 namespacegcworker stdout | 2025-11-04 09:06:56,110 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,111 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,111 [75] [DEBUG] [data.model.storage] Garbage collecting storage 71 namespacegcworker stdout | 2025-11-04 09:06:56,111 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,112 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,113 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71} namespacegcworker stdout | 2025-11-04 09:06:56,113 [75] [DEBUG] [data.model.storage] Garbage collecting storage 48 namespacegcworker stdout | 2025-11-04 09:06:56,113 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,114 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,115 [75] [DEBUG] [data.model.storage] Garbage collecting storage 49 namespacegcworker stdout | 2025-11-04 09:06:56,115 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,117 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,117 [75] [DEBUG] [data.model.storage] Garbage collecting storage 50 namespacegcworker stdout | 2025-11-04 09:06:56,118 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,119 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,119 [75] [DEBUG] [data.model.storage] Garbage collecting storage 51 namespacegcworker stdout | 2025-11-04 09:06:56,119 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,120 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,121 [75] [DEBUG] [data.model.storage] Garbage collecting storage 52 namespacegcworker stdout | 2025-11-04 09:06:56,121 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,122 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,123 [75] [DEBUG] [data.model.storage] Garbage collecting storage 53 namespacegcworker stdout | 2025-11-04 09:06:56,123 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,125 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,126 [75] [DEBUG] [data.model.storage] Garbage collecting storage 54 namespacegcworker stdout | 2025-11-04 09:06:56,126 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,127 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,128 [75] [DEBUG] [data.model.storage] Garbage collecting storage 55 namespacegcworker stdout | 2025-11-04 09:06:56,128 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,129 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,129 [75] [DEBUG] [data.model.storage] Garbage collecting storage 56 namespacegcworker stdout | 2025-11-04 09:06:56,130 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,131 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,131 [75] [DEBUG] [data.model.storage] Garbage collecting storage 57 namespacegcworker stdout | 2025-11-04 09:06:56,131 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,132 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,132 [75] [DEBUG] [data.model.storage] Garbage collecting storage 58 namespacegcworker stdout | 2025-11-04 09:06:56,133 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,133 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,134 [75] [DEBUG] [data.model.storage] Garbage collecting storage 59 namespacegcworker stdout | 2025-11-04 09:06:56,135 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,136 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,136 [75] [DEBUG] [data.model.storage] Garbage collecting storage 60 namespacegcworker stdout | 2025-11-04 09:06:56,136 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,137 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,137 [75] [DEBUG] [data.model.storage] Garbage collecting storage 61 namespacegcworker stdout | 2025-11-04 09:06:56,138 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,138 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,139 [75] [DEBUG] [data.model.storage] Garbage collecting storage 62 namespacegcworker stdout | 2025-11-04 09:06:56,139 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,140 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,140 [75] [DEBUG] [data.model.storage] Garbage collecting storage 63 namespacegcworker stdout | 2025-11-04 09:06:56,140 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,141 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,141 [75] [DEBUG] [data.model.storage] Garbage collecting storage 64 namespacegcworker stdout | 2025-11-04 09:06:56,142 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,142 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,143 [75] [DEBUG] [data.model.storage] Garbage collecting storage 65 namespacegcworker stdout | 2025-11-04 09:06:56,143 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,144 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,145 [75] [DEBUG] [data.model.storage] Garbage collecting storage 66 namespacegcworker stdout | 2025-11-04 09:06:56,145 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,146 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,146 [75] [DEBUG] [data.model.storage] Garbage collecting storage 67 namespacegcworker stdout | 2025-11-04 09:06:56,146 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,147 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,148 [75] [DEBUG] [data.model.storage] Garbage collecting storage 68 namespacegcworker stdout | 2025-11-04 09:06:56,148 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,149 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,149 [75] [DEBUG] [data.model.storage] Garbage collecting storage 69 namespacegcworker stdout | 2025-11-04 09:06:56,149 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,150 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,151 [75] [DEBUG] [data.model.storage] Garbage collecting storage 70 namespacegcworker stdout | 2025-11-04 09:06:56,151 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,152 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,152 [75] [DEBUG] [data.model.storage] Garbage collecting storage 71 namespacegcworker stdout | 2025-11-04 09:06:56,152 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,153 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,154 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,154 [75] [DEBUG] [data.model.gc] Found 2 tags to GC under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,155 [75] [DEBUG] [data.model.gc] Deleting tag 29 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,155 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,156 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-4af04d18-96c0-406a-b8a8-1e3f65488b64']) namespacegcworker stdout | 2025-11-04 09:06:56,156 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,156 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-4af04d18-96c0-406a-b8a8-1e3f65488b64 namespacegcworker stdout | 2025-11-04 09:06:56,156 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,158 [75] [DEBUG] [data.model.gc] Deleting tag 30 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,158 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s FOR UPDATE', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,159 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE (("tagpullstatistics"."repository_id" = %s) AND ("tagpullstatistics"."tag_name" = %s))', [8, '$temp-9e3ef684-096b-49ce-9a7d-a667c3e28147']) namespacegcworker stdout | 2025-11-04 09:06:56,159 [75] [DEBUG] [peewee] ('DELETE FROM "tagnotificationsuccess" WHERE ("tagnotificationsuccess"."tag_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:56,159 [75] [DEBUG] [data.model.notification] Deleted 0 entries from TagNotificationSuccess for tag: $temp-9e3ef684-096b-49ce-9a7d-a667c3e28147 namespacegcworker stdout | 2025-11-04 09:06:56,159 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:56,161 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,162 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,163 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,164 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,165 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,165 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,167 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,168 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,169 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,170 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [29, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,170 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [29, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,171 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [29, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,171 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [29, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,172 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [29, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,172 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,173 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:56,173 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:56,173 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [29, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,174 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:1e5420b375eeab126bf1b2242f8fcf4bc7dae869b58e966dc10bc0c6b22bc0a2']) namespacegcworker stdout | 2025-11-04 09:06:56,174 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [29]) namespacegcworker stdout | 2025-11-04 09:06:56,176 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:1e5420b375eeab126bf1b2242f8fcf4bc7dae869b58e966dc10bc0c6b22bc0a2', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,177 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:56,177 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:1e5420b375eeab126bf1b2242f8fcf4bc7dae869b58e966dc10bc0c6b22bc0a2 namespacegcworker stdout | 2025-11-04 09:06:56,189 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:1e5420b375eeab126bf1b2242f8fcf4bc7dae869b58e966dc10bc0c6b22bc0a2 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,189 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:1e5420b375eeab126bf1b2242f8fcf4bc7dae869b58e966dc10bc0c6b22bc0a2 from secscan service namespacegcworker stdout | 2025-11-04 09:06:56,190 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,191 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,192 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,193 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."manifest_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:56,194 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."manifest_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:56,195 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."manifest_id" = %s)', [30]) namespacegcworker stdout | 2025-11-04 09:06:56,195 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,196 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."manifest_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,197 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."child_manifest_id" FROM "manifestchild" AS "t1" WHERE ("t1"."child_manifest_id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,197 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t2"."subject" = "t1"."digest") WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [30, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,198 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE (("manifestlabel"."manifest_id" = %s) AND ("manifestlabel"."repository_id" = %s))', [30, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,199 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE (("manifestchild"."manifest_id" = %s) AND ("manifestchild"."repository_id" = %s))', [30, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,199 [75] [DEBUG] [peewee] ('SELECT "t1"."image_size", "t1"."id" FROM "imagestorage" AS "t1" INNER JOIN "manifestblob" AS "t2" ON ("t1"."id" = "t2"."blob_id") WHERE (("t2"."manifest_id" = %s) AND ("t2"."repository_id" = %s))', [30, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,200 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE (("manifestblob"."manifest_id" = %s) AND ("manifestblob"."repository_id" = %s))', [30, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,200 [75] [DEBUG] [peewee] ('SELECT "t1"."namespace_user_id" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,201 [75] [DEBUG] [peewee] ('SELECT 1 FROM "user" AS "t1" WHERE ((("t1"."id" = %s) AND "t1"."enabled") AND NOT "t1"."robot") LIMIT %s', [5, 1]) namespacegcworker stdout | 2025-11-04 09:06:56,201 [75] [DEBUG] [data.model.quota] ineligible namespace 5 for quota calculation, skipping calculation namespacegcworker stdout | 2025-11-04 09:06:56,201 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE (("manifestsecuritystatus"."manifest_id" = %s) AND ("manifestsecuritystatus"."repository_id" = %s))', [30, 8]) namespacegcworker stdout | 2025-11-04 09:06:56,202 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE (("manifestpullstatistics"."repository_id" = %s) AND ("manifestpullstatistics"."manifest_digest" = %s))', [8, 'sha256:291c2310bc71c4729898622ebcac75b761af88817542024404788b3681e92dc9']) gunicorn-secscan stdout | 2025-11-04 09:06:56,202 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:06:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" namespacegcworker stdout | 2025-11-04 09:06:56,202 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."id" = %s)', [30]) nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:06:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) namespacegcworker stdout | 2025-11-04 09:06:56,204 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."digest" = %s) LIMIT %s OFFSET %s', ['sha256:291c2310bc71c4729898622ebcac75b761af88817542024404788b3681e92dc9', 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,205 [75] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request namespacegcworker stdout | 2025-11-04 09:06:56,205 [75] [DEBUG] [util.secscan.v4.api] DELETEing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_report/sha256:291c2310bc71c4729898622ebcac75b761af88817542024404788b3681e92dc9 namespacegcworker stdout | 2025-11-04 09:06:56,210 [75] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "DELETE /indexer/api/v1/index_report/sha256:291c2310bc71c4729898622ebcac75b761af88817542024404788b3681e92dc9 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,210 [75] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:291c2310bc71c4729898622ebcac75b761af88817542024404788b3681e92dc9 from secscan service namespacegcworker stdout | 2025-11-04 09:06:56,211 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [96, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,212 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [96]) namespacegcworker stdout | 2025-11-04 09:06:56,213 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [97, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,214 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [97]) namespacegcworker stdout | 2025-11-04 09:06:56,215 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [98, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,217 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [98]) namespacegcworker stdout | 2025-11-04 09:06:56,218 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [92, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,219 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [92]) namespacegcworker stdout | 2025-11-04 09:06:56,220 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [93, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,221 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [93]) namespacegcworker stdout | 2025-11-04 09:06:56,223 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [94, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,223 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [94]) namespacegcworker stdout | 2025-11-04 09:06:56,225 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."label_id" FROM "manifestlabel" AS "t1" WHERE ("t1"."label_id" = %s) LIMIT %s OFFSET %s', [95, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,226 [75] [DEBUG] [peewee] ('DELETE FROM "label" WHERE ("label"."id" = %s)', [95]) namespacegcworker stdout | 2025-11-04 09:06:56,227 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {72, 73, 74, 75, 76} namespacegcworker stdout | 2025-11-04 09:06:56,227 [75] [DEBUG] [data.model.storage] Garbage collecting storage 72 namespacegcworker stdout | 2025-11-04 09:06:56,228 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,228 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,229 [75] [DEBUG] [data.model.storage] Garbage collecting storage 73 namespacegcworker stdout | 2025-11-04 09:06:56,229 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,230 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,231 [75] [DEBUG] [data.model.storage] Garbage collecting storage 74 namespacegcworker stdout | 2025-11-04 09:06:56,231 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,232 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,233 [75] [DEBUG] [data.model.storage] Garbage collecting storage 75 namespacegcworker stdout | 2025-11-04 09:06:56,233 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,234 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,235 [75] [DEBUG] [data.model.storage] Garbage collecting storage 76 namespacegcworker stdout | 2025-11-04 09:06:56,235 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,236 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,237 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {72, 73, 74, 75, 76} namespacegcworker stdout | 2025-11-04 09:06:56,237 [75] [DEBUG] [data.model.storage] Garbage collecting storage 72 namespacegcworker stdout | 2025-11-04 09:06:56,237 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,238 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,239 [75] [DEBUG] [data.model.storage] Garbage collecting storage 73 namespacegcworker stdout | 2025-11-04 09:06:56,239 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,240 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,241 [75] [DEBUG] [data.model.storage] Garbage collecting storage 74 namespacegcworker stdout | 2025-11-04 09:06:56,241 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,242 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,243 [75] [DEBUG] [data.model.storage] Garbage collecting storage 75 namespacegcworker stdout | 2025-11-04 09:06:56,243 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,244 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,244 [75] [DEBUG] [data.model.storage] Garbage collecting storage 76 namespacegcworker stdout | 2025-11-04 09:06:56,245 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,246 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,247 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,248 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id" FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,249 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,250 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,250 [75] [DEBUG] [data.model.gc] Found 10 uploaded blobs to GC under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,251 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 47 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,251 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [47]) namespacegcworker stdout | 2025-11-04 09:06:56,252 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 48 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,253 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:56,254 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 49 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,254 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:56,256 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 50 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,256 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:56,258 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 51 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,258 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:56,259 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 52 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,260 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:56,261 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 53 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,262 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:56,263 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 54 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,263 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:56,264 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 55 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,265 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:56,266 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 56 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,266 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:56,268 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {48, 49, 50, 51, 52, 53, 54, 55, 56, 57} namespacegcworker stdout | 2025-11-04 09:06:56,268 [75] [DEBUG] [data.model.storage] Garbage collecting storage 48 namespacegcworker stdout | 2025-11-04 09:06:56,268 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,270 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [48, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,271 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:56,272 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:56,272 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:56,273 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [48]) namespacegcworker stdout | 2025-11-04 09:06:56,274 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b']) namespacegcworker stdout | 2025-11-04 09:06:56,274 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b']) namespacegcworker stdout | 2025-11-04 09:06:56,276 [75] [DEBUG] [data.model.storage] Garbage collecting storage 49 namespacegcworker stdout | 2025-11-04 09:06:56,276 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,278 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [49, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,279 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:56,279 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:56,280 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:56,280 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [49]) namespacegcworker stdout | 2025-11-04 09:06:56,281 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38']) namespacegcworker stdout | 2025-11-04 09:06:56,282 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38']) namespacegcworker stdout | 2025-11-04 09:06:56,283 [75] [DEBUG] [data.model.storage] Garbage collecting storage 50 namespacegcworker stdout | 2025-11-04 09:06:56,284 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,284 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,286 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:56,286 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:56,287 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:56,287 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [50]) namespacegcworker stdout | 2025-11-04 09:06:56,288 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410']) namespacegcworker stdout | 2025-11-04 09:06:56,288 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410']) namespacegcworker stdout | 2025-11-04 09:06:56,290 [75] [DEBUG] [data.model.storage] Garbage collecting storage 51 namespacegcworker stdout | 2025-11-04 09:06:56,290 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,291 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [51, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,292 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:56,293 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:56,293 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:56,294 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [51]) namespacegcworker stdout | 2025-11-04 09:06:56,295 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca']) namespacegcworker stdout | 2025-11-04 09:06:56,295 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca']) namespacegcworker stdout | 2025-11-04 09:06:56,297 [75] [DEBUG] [data.model.storage] Garbage collecting storage 52 namespacegcworker stdout | 2025-11-04 09:06:56,297 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,299 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [52, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,300 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:56,300 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:56,301 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:56,301 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [52]) namespacegcworker stdout | 2025-11-04 09:06:56,302 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576']) namespacegcworker stdout | 2025-11-04 09:06:56,303 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576']) namespacegcworker stdout | 2025-11-04 09:06:56,304 [75] [DEBUG] [data.model.storage] Garbage collecting storage 53 namespacegcworker stdout | 2025-11-04 09:06:56,304 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,305 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [53, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,306 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:56,307 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:56,308 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:56,309 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [53]) namespacegcworker stdout | 2025-11-04 09:06:56,309 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0']) namespacegcworker stdout | 2025-11-04 09:06:56,310 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0']) namespacegcworker stdout | 2025-11-04 09:06:56,312 [75] [DEBUG] [data.model.storage] Garbage collecting storage 54 namespacegcworker stdout | 2025-11-04 09:06:56,312 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,313 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [54, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,315 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:56,315 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:56,316 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:56,316 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [54]) namespacegcworker stdout | 2025-11-04 09:06:56,317 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d']) namespacegcworker stdout | 2025-11-04 09:06:56,318 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d']) namespacegcworker stdout | 2025-11-04 09:06:56,319 [75] [DEBUG] [data.model.storage] Garbage collecting storage 55 namespacegcworker stdout | 2025-11-04 09:06:56,320 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,321 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [55, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,322 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:56,322 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:56,323 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:56,323 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [55]) namespacegcworker stdout | 2025-11-04 09:06:56,324 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b']) namespacegcworker stdout | 2025-11-04 09:06:56,324 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b']) namespacegcworker stdout | 2025-11-04 09:06:56,326 [75] [DEBUG] [data.model.storage] Garbage collecting storage 56 namespacegcworker stdout | 2025-11-04 09:06:56,326 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,327 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [56, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,328 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:56,329 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:56,329 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:56,330 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [56]) namespacegcworker stdout | 2025-11-04 09:06:56,331 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837']) namespacegcworker stdout | 2025-11-04 09:06:56,331 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837']) namespacegcworker stdout | 2025-11-04 09:06:56,333 [75] [DEBUG] [data.model.storage] Garbage collecting storage 57 namespacegcworker stdout | 2025-11-04 09:06:56,333 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,334 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [57, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,335 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:56,336 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:56,336 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:56,337 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:56,337 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c']) namespacegcworker stdout | 2025-11-04 09:06:56,338 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c']) namespacegcworker stdout | 2025-11-04 09:06:56,340 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,340 [75] [DEBUG] [data.model.storage] Removing sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,340 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,341 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576'} namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,342 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,343 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 323cbe077a6e99e9dbb8ec02b4e7baa158f68525ecba44f6ca1b710dd8d5d6ad namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 67442f62937430f69bd7655bf4110825c7dc139dbcd7d9a1d972c897106ad0a6 namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=67442f62937430f69bd7655bf4110825c7dc139dbcd7d9a1d972c897106ad0a6', 'amz-sdk-invocation-id': b'0676aef7-fc01-458d-864a-fe869e7fc485', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,344 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,349 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchql6-8jzdzl-971', 'x-amz-id-2': 'mhkchql6-8jzdzl-971', 'ETag': '"b47984438ca833092da49d2304cca86a-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:18 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,350 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchql6-8jzdzl-971', 'HostId': 'mhkchql6-8jzdzl-971', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchql6-8jzdzl-971', 'x-amz-id-2': 'mhkchql6-8jzdzl-971', 'etag': '"b47984438ca833092da49d2304cca86a-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:18 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 18, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"b47984438ca833092da49d2304cca86a-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576'} namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,351 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 67e10b37296d023a6e6ca2cb9113ad0fe078acead0de285386208f4eeb921756 namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | b60fb3bc5fd594cdaf5a28b0830a74716c6463114491d3319b0b02cce1323594 namespacegcworker stdout | 2025-11-04 09:06:56,352 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,353 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,353 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,353 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=b60fb3bc5fd594cdaf5a28b0830a74716c6463114491d3319b0b02cce1323594', 'amz-sdk-invocation-id': b'e7be6556-9c84-41e5-8ba1-582c8eb27fbe', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,353 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,360 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2acaf080abe6a7e9f0f9a5420a21dca46fa437713f9743a33942040e07b8c576 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,360 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqle-8p0e4q-1dlj', 'x-amz-id-2': 'mhkchqle-8p0e4q-1dlj', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,360 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,360 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,361 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,361 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,361 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,361 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqle-8p0e4q-1dlj', 'HostId': 'mhkchqle-8p0e4q-1dlj', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqle-8p0e4q-1dlj', 'x-amz-id-2': 'mhkchqle-8p0e4q-1dlj', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,362 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,362 [75] [DEBUG] [data.model.storage] Removing sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d from local_us namespacegcworker stdout | 2025-11-04 09:06:56,362 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,363 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,363 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d'} namespacegcworker stdout | 2025-11-04 09:06:56,363 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,364 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 8ab043740ab87bb6705fad0e500e48604ca7a4cf8046cba984decb1dad30b620 namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1841cb3b00f4e063d706c6cb3fcd977cb48fa2dcbd6f7325f725eeadfe8f9e53 namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,365 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,366 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,366 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1841cb3b00f4e063d706c6cb3fcd977cb48fa2dcbd6f7325f725eeadfe8f9e53', 'amz-sdk-invocation-id': b'42c82b0b-4186-4af4-bd92-cde832386651', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,366 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,370 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqlr-8wfy9d-1503', 'x-amz-id-2': 'mhkchqlr-8wfy9d-1503', 'ETag': '"7fedfa9a207a482d135bc7ed6b2fc247-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:20 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '593', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,371 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqlr-8wfy9d-1503', 'HostId': 'mhkchqlr-8wfy9d-1503', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqlr-8wfy9d-1503', 'x-amz-id-2': 'mhkchqlr-8wfy9d-1503', 'etag': '"7fedfa9a207a482d135bc7ed6b2fc247-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:20 GMT', 'content-type': 'application/octet-stream', 'content-length': '593', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 20, tzinfo=tzutc()), 'ContentLength': 593, 'ETag': '"7fedfa9a207a482d135bc7ed6b2fc247-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d'} namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,372 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 161851cf36cf47b7b908e7c529f497a920b8aeafd3381459f8ff1db7f029dba5 namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3b5228d895f242b9e18522ab5eb24884ac729ef53267cbb29e3bb3612427293c namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,373 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3b5228d895f242b9e18522ab5eb24884ac729ef53267cbb29e3bb3612427293c', 'amz-sdk-invocation-id': b'8e070e75-466e-4d26-86b6-d8dcdb25f562', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,374 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,379 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/09/092cf4c3899032cdf45f3159ec6959b7ad1039335b892f4e0737f07f36eadd8d HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,379 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqlz-913q8x-t4a', 'x-amz-id-2': 'mhkchqlz-913q8x-t4a', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,379 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqlz-913q8x-t4a', 'HostId': 'mhkchqlz-913q8x-t4a', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqlz-913q8x-t4a', 'x-amz-id-2': 'mhkchqlz-913q8x-t4a', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,380 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,381 [75] [DEBUG] [data.model.storage] Removing sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b from local_us namespacegcworker stdout | 2025-11-04 09:06:56,381 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b'} namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,382 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 548cddd3bdb4486e3bb9d0f4dd8ea4ade34e918e5877eab330c91d1d30eb65dc namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | e2642e8d5d6ccd90fc7ffc20aae1160cf717753967efec5237e331ad12d9fc93 namespacegcworker stdout | 2025-11-04 09:06:56,383 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,384 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,384 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,384 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e2642e8d5d6ccd90fc7ffc20aae1160cf717753967efec5237e331ad12d9fc93', 'amz-sdk-invocation-id': b'0765ea06-00ae-4926-89e3-5bc033f5fa46', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,384 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,387 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,387 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqm9-9754zq-zpa', 'x-amz-id-2': 'mhkchqm9-9754zq-zpa', 'ETag': '"68416394ec38f57ed50d73a5eae98612-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:15 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3802452', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,387 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqm9-9754zq-zpa', 'HostId': 'mhkchqm9-9754zq-zpa', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqm9-9754zq-zpa', 'x-amz-id-2': 'mhkchqm9-9754zq-zpa', 'etag': '"68416394ec38f57ed50d73a5eae98612-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:15 GMT', 'content-type': 'application/octet-stream', 'content-length': '3802452', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 15, tzinfo=tzutc()), 'ContentLength': 3802452, 'ETag': '"68416394ec38f57ed50d73a5eae98612-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b'} namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,388 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 88b790ae55718aa4b9851dddc211f7c3543d5526c1e9cb994ea5a45c564eca01 namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | dba8c777b358d1adbd7c7a88accb40c19ce5c466d2a098d5315f4b89fb636e88 namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,389 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=dba8c777b358d1adbd7c7a88accb40c19ce5c466d2a098d5315f4b89fb636e88', 'amz-sdk-invocation-id': b'f0d7f0cc-31ed-45e6-b19b-f27c602a6c41', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,390 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,395 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2d/2d35ebdb57d9971fea0cac1582aa78935adf8058b2cc32db163c98822e5dfa1b HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,395 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqme-9akalh-182n', 'x-amz-id-2': 'mhkchqme-9akalh-182n', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,395 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqme-9akalh-182n', 'HostId': 'mhkchqme-9akalh-182n', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqme-9akalh-182n', 'x-amz-id-2': 'mhkchqme-9akalh-182n', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,396 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,397 [75] [DEBUG] [data.model.storage] Removing sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c from local_us namespacegcworker stdout | 2025-11-04 09:06:56,397 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c'} namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,398 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0bf88cfd40fc6c0e84b9a841778d81bfc4a90335e606ead41c5be79ce82a1886 namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 14de9f04d83f049b85c1a534d0fb60537e7403538508edfd3a5a8ec9312566ef namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=14de9f04d83f049b85c1a534d0fb60537e7403538508edfd3a5a8ec9312566ef', 'amz-sdk-invocation-id': b'2eb9d348-1e51-4bed-9306-ef2d714f1dc9', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,399 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,403 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,403 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqmo-9gg5ig-asl', 'x-amz-id-2': 'mhkchqmo-9gg5ig-asl', 'ETag': '"3e950416717efd3beaa626d559dfbcfe-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:23 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3221555', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqmo-9gg5ig-asl', 'HostId': 'mhkchqmo-9gg5ig-asl', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqmo-9gg5ig-asl', 'x-amz-id-2': 'mhkchqmo-9gg5ig-asl', 'etag': '"3e950416717efd3beaa626d559dfbcfe-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:23 GMT', 'content-type': 'application/octet-stream', 'content-length': '3221555', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 23, tzinfo=tzutc()), 'ContentLength': 3221555, 'ETag': '"3e950416717efd3beaa626d559dfbcfe-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c'} namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,404 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 77561a3a0981978d4ba594d8a5549ac83bde421f435d7c66e085e9b5ef7438ed namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5c7524486ecd373db4f2b6c686526d7023d6ebf240e68d38ff47e436128dafdb namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,405 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5c7524486ecd373db4f2b6c686526d7023d6ebf240e68d38ff47e436128dafdb', 'amz-sdk-invocation-id': b'd5a9304b-5c79-4ffa-a300-ae75a13a08f7', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,406 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/27/2763c7fc79b66030222442365f4a0f69d9dbaa11f7fd47a918d29d732d52996c HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqmu-9k2i8x-j1u', 'x-amz-id-2': 'mhkchqmu-9k2i8x-j1u', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,412 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqmu-9k2i8x-j1u', 'HostId': 'mhkchqmu-9k2i8x-j1u', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqmu-9k2i8x-j1u', 'x-amz-id-2': 'mhkchqmu-9k2i8x-j1u', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,413 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,413 [75] [DEBUG] [data.model.storage] Removing sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,413 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38'} namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,414 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,415 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | c5b837945390ded2f243206b2428a54b5a1d8b1296181dee3e7dac675fc5833f namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1a6a2369ffb5875f4dba6d92b07fa11271dc719d79eee98fdf211f519b087771 namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1a6a2369ffb5875f4dba6d92b07fa11271dc719d79eee98fdf211f519b087771', 'amz-sdk-invocation-id': b'7c648dba-280e-4336-b5a1-171467d332a6', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,416 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,420 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,420 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqn5-9q9uix-mte', 'x-amz-id-2': 'mhkchqn5-9q9uix-mte', 'ETag': '"16944cf0cd96381339fcfbbf68c84751-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:16 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '581', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,420 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,421 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,421 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,421 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,421 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqn5-9q9uix-mte', 'HostId': 'mhkchqn5-9q9uix-mte', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqn5-9q9uix-mte', 'x-amz-id-2': 'mhkchqn5-9q9uix-mte', 'etag': '"16944cf0cd96381339fcfbbf68c84751-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:16 GMT', 'content-type': 'application/octet-stream', 'content-length': '581', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 16, tzinfo=tzutc()), 'ContentLength': 581, 'ETag': '"16944cf0cd96381339fcfbbf68c84751-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38'} namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,422 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 9f2a9a4eb06214a6fd75b0730d604bee56257a2f891fa411ffdcfbe954c9a212 namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6aed5329e28e706d0029f2dd195f05cd2304d5facf8f4de0c7f91e6f868efac7 namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6aed5329e28e706d0029f2dd195f05cd2304d5facf8f4de0c7f91e6f868efac7', 'amz-sdk-invocation-id': b'7376c86c-e767-4bdf-9785-4f97120ab311', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,423 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/70/706db57fb2063f39f69632c5b5c9c439633fda35110e65587c5d85553fd1cc38 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnc-9ulphq-50o', 'x-amz-id-2': 'mhkchqnc-9ulphq-50o', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqnc-9ulphq-50o', 'HostId': 'mhkchqnc-9ulphq-50o', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnc-9ulphq-50o', 'x-amz-id-2': 'mhkchqnc-9ulphq-50o', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,429 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,430 [75] [DEBUG] [data.model.storage] Removing sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,430 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410'} namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,431 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 258813c82f5698e795b899558a288cc046530225f151f51c8cf1931e06fb8895 namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | d6de8f7972ad92b91cf313795d7a2855ac0d02553e66786984e2f55da250de00 namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d6de8f7972ad92b91cf313795d7a2855ac0d02553e66786984e2f55da250de00', 'amz-sdk-invocation-id': b'e18162fc-6d23-4476-b01b-d8c0aa9f66ce', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,432 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnl-9zzqbf-1br7', 'x-amz-id-2': 'mhkchqnl-9zzqbf-1br7', 'ETag': '"3e100f98a7e0aa6adce14a518ec38587-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:17 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80956', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,436 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqnl-9zzqbf-1br7', 'HostId': 'mhkchqnl-9zzqbf-1br7', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnl-9zzqbf-1br7', 'x-amz-id-2': 'mhkchqnl-9zzqbf-1br7', 'etag': '"3e100f98a7e0aa6adce14a518ec38587-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:17 GMT', 'content-type': 'application/octet-stream', 'content-length': '80956', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 17, tzinfo=tzutc()), 'ContentLength': 80956, 'ETag': '"3e100f98a7e0aa6adce14a518ec38587-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410'} namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,437 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0062e54b0fafe8fee4a59726cd349757a373c1c87972df80eaffd2960ded9d5f namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 90d9290011d580e76a96db5dfc8f53ebb70942543bf1646a02adb7db577dd1e4 namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=90d9290011d580e76a96db5dfc8f53ebb70942543bf1646a02adb7db577dd1e4', 'amz-sdk-invocation-id': b'8a354e1a-66c4-4fb6-ab5e-3f0a6495a626', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,438 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/40/40ad67084d10a89012157e0e89a71b883fc9b0fe0b4a944b8f7d39b92d1bc410 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnr-a3fpa6-o3t', 'x-amz-id-2': 'mhkchqnr-a3fpa6-o3t', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqnr-a3fpa6-o3t', 'HostId': 'mhkchqnr-a3fpa6-o3t', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqnr-a3fpa6-o3t', 'x-amz-id-2': 'mhkchqnr-a3fpa6-o3t', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,445 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,446 [75] [DEBUG] [data.model.storage] Removing sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca from local_us namespacegcworker stdout | 2025-11-04 09:06:56,446 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca'} namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,447 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f64e0db9ad3dcd57cf57ad1418a929bbe0a35ab77e72e679b840cb6a1c53885a namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 9028729f49d6cb33caf98b3b77cc0ec41d40556400d58d9bef205eee8288592f namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,448 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,449 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,449 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9028729f49d6cb33caf98b3b77cc0ec41d40556400d58d9bef205eee8288592f', 'amz-sdk-invocation-id': b'1ad37157-483e-4232-be50-2a2a96c696eb', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,449 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,452 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqo2-a9top7-ec6', 'x-amz-id-2': 'mhkchqo2-a9top7-ec6', 'ETag': '"6c05f48e52d9029f03e26717bf5e4e7e-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:17 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5682', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqo2-a9top7-ec6', 'HostId': 'mhkchqo2-a9top7-ec6', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqo2-a9top7-ec6', 'x-amz-id-2': 'mhkchqo2-a9top7-ec6', 'etag': '"6c05f48e52d9029f03e26717bf5e4e7e-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:17 GMT', 'content-type': 'application/octet-stream', 'content-length': '5682', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 17, tzinfo=tzutc()), 'ContentLength': 5682, 'ETag': '"6c05f48e52d9029f03e26717bf5e4e7e-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca'} namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,453 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 677b17b5dad784a1dde9aaa44cb49a9f151a5ccf542aee235f869ab62926331d namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6bcf800070974f891847063d6b4a32b7153dfb6d8c39bb3fa188058db9689130 namespacegcworker stdout | 2025-11-04 09:06:56,454 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,455 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,455 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,455 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6bcf800070974f891847063d6b4a32b7153dfb6d8c39bb3fa188058db9689130', 'amz-sdk-invocation-id': b'3a773410-2415-4da5-aa0b-aeb7f82fe858', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,455 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/eb/ebbebd44c17fcfc36f52ae1aef89bc9d1eaec8a2d865362f62bcce19e2fcaaca HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqo8-adnrwf-15ql', 'x-amz-id-2': 'mhkchqo8-adnrwf-15ql', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,461 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqo8-adnrwf-15ql', 'HostId': 'mhkchqo8-adnrwf-15ql', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqo8-adnrwf-15ql', 'x-amz-id-2': 'mhkchqo8-adnrwf-15ql', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,462 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,462 [75] [DEBUG] [data.model.storage] Removing sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,462 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0'} namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,463 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | fc1761c083bff4f22ff4ed1854d797951ea64863030206024cd2c26d2791a0f9 namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | d83d8003152d9b63153b59bc50e87f7a39249b2ee82157c8a5116fd008e1d8c2 namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,464 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,465 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d83d8003152d9b63153b59bc50e87f7a39249b2ee82157c8a5116fd008e1d8c2', 'amz-sdk-invocation-id': b'dbc84522-2df8-41e9-ac52-0175aeebc77e', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,465 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,468 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqoi-ajbq0s-lsc', 'x-amz-id-2': 'mhkchqoi-ajbq0s-lsc', 'ETag': '"4f2f8a87dda82df9ede7dba4a9a27db9-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:19 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3504080', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqoi-ajbq0s-lsc', 'HostId': 'mhkchqoi-ajbq0s-lsc', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqoi-ajbq0s-lsc', 'x-amz-id-2': 'mhkchqoi-ajbq0s-lsc', 'etag': '"4f2f8a87dda82df9ede7dba4a9a27db9-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:19 GMT', 'content-type': 'application/octet-stream', 'content-length': '3504080', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 19, tzinfo=tzutc()), 'ContentLength': 3504080, 'ETag': '"4f2f8a87dda82df9ede7dba4a9a27db9-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0'} namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,469 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b5c21220879de980e74b6b50e470995c88235e2660268bc5329acb57fe7d8f82 namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 3d0731298fcf4ebe342697aee1acdfe31436e0eee40e40aaa4331a8d63253569 namespacegcworker stdout | 2025-11-04 09:06:56,470 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,471 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,471 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,471 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3d0731298fcf4ebe342697aee1acdfe31436e0eee40e40aaa4331a8d63253569', 'amz-sdk-invocation-id': b'a6b6ea68-0d6e-42ae-ad0e-0cb47be17a96', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,471 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,476 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/bb/bb1da3d879939be7df9f182950d2fb201d4fc2e1043677da2037cd6afb084ce0 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqoo-amxeto-6p5', 'x-amz-id-2': 'mhkchqoo-amxeto-6p5', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqoo-amxeto-6p5', 'HostId': 'mhkchqoo-amxeto-6p5', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqoo-amxeto-6p5', 'x-amz-id-2': 'mhkchqoo-amxeto-6p5', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,477 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [data.model.storage] Removing sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837'} namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,478 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 29a94b9b64cd63fd5da2bd79c3fa2072ee1e79ba2dbcecbbbb561332fbf2508f namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | aab702fe2b5988344299f372fd7d2f239fb18e896cf590808139dabc88fc4a47 namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,479 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,480 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,480 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=aab702fe2b5988344299f372fd7d2f239fb18e896cf590808139dabc88fc4a47', 'amz-sdk-invocation-id': b'3abfd3ea-34ad-4ea0-a89e-0a13d80d4793', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,480 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,483 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqox-asfuic-11yx', 'x-amz-id-2': 'mhkchqox-asfuic-11yx', 'ETag': '"17a5224acd8c7fe3de8a773aa545f1da-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:21 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '167', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqox-asfuic-11yx', 'HostId': 'mhkchqox-asfuic-11yx', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqox-asfuic-11yx', 'x-amz-id-2': 'mhkchqox-asfuic-11yx', 'etag': '"17a5224acd8c7fe3de8a773aa545f1da-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:21 GMT', 'content-type': 'application/octet-stream', 'content-length': '167', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 21, tzinfo=tzutc()), 'ContentLength': 167, 'ETag': '"17a5224acd8c7fe3de8a773aa545f1da-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837'} namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,484 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 21a0b70cc75534cb73de31926c6fa08a9ff3bd40fc111a6d6a0d8dea064f5ea7 namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 8dbaa2dcaf73718460e20a63dc3899652b3ce69bcda5948208e39cd1592c8409 namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,485 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8dbaa2dcaf73718460e20a63dc3899652b3ce69bcda5948208e39cd1592c8409', 'amz-sdk-invocation-id': b'4d07ac28-5cd6-48fb-9383-0adca5ba1f30', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,486 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/ac/acf9bc9d7b6017b5062a78ea1f7634a1310e37c1f1f03fd4e27afea1bc437837 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqp2-avp03x-9rm', 'x-amz-id-2': 'mhkchqp2-avp03x-9rm', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqp2-avp03x-9rm', 'HostId': 'mhkchqp2-avp03x-9rm', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqp2-avp03x-9rm', 'x-amz-id-2': 'mhkchqp2-avp03x-9rm', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,492 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,493 [75] [DEBUG] [data.model.storage] Removing sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b from local_us namespacegcworker stdout | 2025-11-04 09:06:56,493 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b'} namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,494 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | a719006a9d1dd25d068b964149f51c3d32be9f3e2191227ac16b8e345896e37a namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 31c42c0713b9db67e62713d699e75a4361826c1f40ae868bce41111d8cf5b0d0 namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,495 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,496 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,496 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=31c42c0713b9db67e62713d699e75a4361826c1f40ae868bce41111d8cf5b0d0', 'amz-sdk-invocation-id': b'de3b8548-f8e4-4301-be10-d1aa39ee802d', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,496 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,499 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,499 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqpd-b1qfg6-g0f', 'x-amz-id-2': 'mhkchqpd-b1qfg6-g0f', 'ETag': '"e89c56a4c4f73f1ed5fa85f2c65a8327-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:21 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5508', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,499 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqpd-b1qfg6-g0f', 'HostId': 'mhkchqpd-b1qfg6-g0f', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqpd-b1qfg6-g0f', 'x-amz-id-2': 'mhkchqpd-b1qfg6-g0f', 'etag': '"e89c56a4c4f73f1ed5fa85f2c65a8327-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:21 GMT', 'content-type': 'application/octet-stream', 'content-length': '5508', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 21, tzinfo=tzutc()), 'ContentLength': 5508, 'ETag': '"e89c56a4c4f73f1ed5fa85f2c65a8327-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b'} namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,500 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | ef2cfdc1660c6dcc55851ac5f8ca9dae32d8d27e36faf3de82a831c24a44b1c9 namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 7acd218b5a6eb407a140955759033d3c2b14234840f112c45e0e96646e6e4acf namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7acd218b5a6eb407a140955759033d3c2b14234840f112c45e0e96646e6e4acf', 'amz-sdk-invocation-id': b'84baa75f-7887-40f1-bcf2-f2b2ba776b39', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,501 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,507 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1c/1ca2771610fc6d5b38d478d41ddd531d6335c5e60afbdb05bd4d8286aaa5276b HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,507 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqpi-b52zz4-15ot', 'x-amz-id-2': 'mhkchqpi-b52zz4-15ot', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,507 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqpi-b52zz4-15ot', 'HostId': 'mhkchqpi-b52zz4-15ot', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqpi-b52zz4-15ot', 'x-amz-id-2': 'mhkchqpi-b52zz4-15ot', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,508 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,509 [75] [DEBUG] [data.model.gc] Found 10 uploaded blobs to GC under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,509 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 57 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,510 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [57]) namespacegcworker stdout | 2025-11-04 09:06:56,511 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 58 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,511 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:56,512 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 59 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,513 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:56,514 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 60 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,514 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,516 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 61 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,517 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,518 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 62 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,518 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,519 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 63 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,520 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,521 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 64 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,521 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,522 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 65 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,523 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,524 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 66 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,524 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,525 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {64, 65, 66, 67, 58, 59, 60, 61, 62, 63} namespacegcworker stdout | 2025-11-04 09:06:56,525 [75] [DEBUG] [data.model.storage] Garbage collecting storage 64 namespacegcworker stdout | 2025-11-04 09:06:56,526 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,526 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [64, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,527 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,528 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,528 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,529 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [64]) namespacegcworker stdout | 2025-11-04 09:06:56,529 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af']) namespacegcworker stdout | 2025-11-04 09:06:56,529 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af']) namespacegcworker stdout | 2025-11-04 09:06:56,531 [75] [DEBUG] [data.model.storage] Garbage collecting storage 65 namespacegcworker stdout | 2025-11-04 09:06:56,531 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,532 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [65, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,533 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,533 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,534 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,534 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [65]) namespacegcworker stdout | 2025-11-04 09:06:56,534 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91']) namespacegcworker stdout | 2025-11-04 09:06:56,535 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91']) namespacegcworker stdout | 2025-11-04 09:06:56,536 [75] [DEBUG] [data.model.storage] Garbage collecting storage 66 namespacegcworker stdout | 2025-11-04 09:06:56,536 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,537 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [66, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,538 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,538 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,539 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,539 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [66]) namespacegcworker stdout | 2025-11-04 09:06:56,540 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531']) namespacegcworker stdout | 2025-11-04 09:06:56,540 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531']) namespacegcworker stdout | 2025-11-04 09:06:56,541 [75] [DEBUG] [data.model.storage] Garbage collecting storage 67 namespacegcworker stdout | 2025-11-04 09:06:56,542 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,542 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [67, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,543 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,544 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,544 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,545 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,545 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e']) namespacegcworker stdout | 2025-11-04 09:06:56,545 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e']) namespacegcworker stdout | 2025-11-04 09:06:56,547 [75] [DEBUG] [data.model.storage] Garbage collecting storage 58 namespacegcworker stdout | 2025-11-04 09:06:56,548 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,548 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [58, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,549 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:56,550 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:56,550 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:56,550 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [58]) namespacegcworker stdout | 2025-11-04 09:06:56,551 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719']) namespacegcworker stdout | 2025-11-04 09:06:56,551 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719']) namespacegcworker stdout | 2025-11-04 09:06:56,552 [75] [DEBUG] [data.model.storage] Garbage collecting storage 59 namespacegcworker stdout | 2025-11-04 09:06:56,553 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,553 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [59, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,554 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:56,555 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:56,556 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:56,556 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [59]) namespacegcworker stdout | 2025-11-04 09:06:56,557 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4']) namespacegcworker stdout | 2025-11-04 09:06:56,557 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4']) namespacegcworker stdout | 2025-11-04 09:06:56,559 [75] [DEBUG] [data.model.storage] Garbage collecting storage 60 namespacegcworker stdout | 2025-11-04 09:06:56,559 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,559 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [60, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,560 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,561 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,561 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,562 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [60]) namespacegcworker stdout | 2025-11-04 09:06:56,562 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6']) namespacegcworker stdout | 2025-11-04 09:06:56,563 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6']) namespacegcworker stdout | 2025-11-04 09:06:56,564 [75] [DEBUG] [data.model.storage] Garbage collecting storage 61 namespacegcworker stdout | 2025-11-04 09:06:56,564 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,565 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [61, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,566 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,566 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,567 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,567 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [61]) namespacegcworker stdout | 2025-11-04 09:06:56,567 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2']) namespacegcworker stdout | 2025-11-04 09:06:56,568 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2']) namespacegcworker stdout | 2025-11-04 09:06:56,569 [75] [DEBUG] [data.model.storage] Garbage collecting storage 62 namespacegcworker stdout | 2025-11-04 09:06:56,570 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,570 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [62, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,571 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,572 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,572 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,572 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [62]) namespacegcworker stdout | 2025-11-04 09:06:56,573 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39']) namespacegcworker stdout | 2025-11-04 09:06:56,573 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39']) namespacegcworker stdout | 2025-11-04 09:06:56,575 [75] [DEBUG] [data.model.storage] Garbage collecting storage 63 namespacegcworker stdout | 2025-11-04 09:06:56,575 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,576 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [63, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,576 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,577 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,577 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,577 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [63]) namespacegcworker stdout | 2025-11-04 09:06:56,578 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc']) namespacegcworker stdout | 2025-11-04 09:06:56,579 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc']) namespacegcworker stdout | 2025-11-04 09:06:56,581 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,582 [75] [DEBUG] [data.model.storage] Removing sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,582 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6'} namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,583 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | cabeb0973ef45d3e38e59a6334d6b4bd034bba5566d8d5c0aa9b5013b1d6a6d0 namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 23670cf7fa2e748ce17980e66a857640e660e3b1fc55d4bea77e1a038dc8a65e namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,584 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=23670cf7fa2e748ce17980e66a857640e660e3b1fc55d4bea77e1a038dc8a65e', 'amz-sdk-invocation-id': b'8a2a98c3-3b22-4e48-957b-800d9c5c2c84', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,585 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-secscan stdout | 2025-11-04 09:06:56,585 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:06:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:06:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqru-cis5u3-5ck', 'x-amz-id-2': 'mhkchqru-cis5u3-5ck', 'ETag': '"1b67366a6a0f38bc42f1f3684b3bcf7f-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:24 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80992', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqru-cis5u3-5ck', 'HostId': 'mhkchqru-cis5u3-5ck', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqru-cis5u3-5ck', 'x-amz-id-2': 'mhkchqru-cis5u3-5ck', 'etag': '"1b67366a6a0f38bc42f1f3684b3bcf7f-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:24 GMT', 'content-type': 'application/octet-stream', 'content-length': '80992', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 24, tzinfo=tzutc()), 'ContentLength': 80992, 'ETag': '"1b67366a6a0f38bc42f1f3684b3bcf7f-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,590 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6'} namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,591 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 86f07d1a396b53664ac5d29cb414e785114e0b51c159669c9f624a2eb5186582 namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 52aebf779e5fea8174e185560923b6baf83b341a00f2a2d96bbf3cdcc5aff883 namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=52aebf779e5fea8174e185560923b6baf83b341a00f2a2d96bbf3cdcc5aff883', 'amz-sdk-invocation-id': b'077e7969-4acd-4e02-8df8-32944c706639', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,592 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/c1/c1504a200153a9b08998d4bd1c0556a2d286bbed359769959ed1bf29b1408fe6 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqs1-cn4wis-14wl', 'x-amz-id-2': 'mhkchqs1-cn4wis-14wl', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,598 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,599 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqs1-cn4wis-14wl', 'HostId': 'mhkchqs1-cn4wis-14wl', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqs1-cn4wis-14wl', 'x-amz-id-2': 'mhkchqs1-cn4wis-14wl', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,599 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,599 [75] [DEBUG] [data.model.storage] Removing sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e from local_us namespacegcworker stdout | 2025-11-04 09:06:56,599 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e'} namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,600 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b8b82f2fcd42c7c97a445868d5b6b062d21a8f9936b0f0defc9343b92831cefa namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 2377ec951cbc596b3c6ef25864067864c07ae38856d8349eafaf4acdc2849cb2 namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,601 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2377ec951cbc596b3c6ef25864067864c07ae38856d8349eafaf4acdc2849cb2', 'amz-sdk-invocation-id': b'1723fb33-ba33-43ed-8de0-da6f5716dac2', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,602 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqsb-ct1d6m-19ue', 'x-amz-id-2': 'mhkchqsb-ct1d6m-19ue', 'ETag': '"4dd07cb0454f1a03ffa03d19450e2474-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:30 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3618931', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,606 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqsb-ct1d6m-19ue', 'HostId': 'mhkchqsb-ct1d6m-19ue', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqsb-ct1d6m-19ue', 'x-amz-id-2': 'mhkchqsb-ct1d6m-19ue', 'etag': '"4dd07cb0454f1a03ffa03d19450e2474-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:30 GMT', 'content-type': 'application/octet-stream', 'content-length': '3618931', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 30, tzinfo=tzutc()), 'ContentLength': 3618931, 'ETag': '"4dd07cb0454f1a03ffa03d19450e2474-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e'} namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,607 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0c1eed4b7bdb779c7a7483ef316cb41b485adf2a412afed47e5fc2a136920d04 namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | ded63563f95e31de39c43e98b41db56670b33ed2557781dd6fb84ed199037599 namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ded63563f95e31de39c43e98b41db56670b33ed2557781dd6fb84ed199037599', 'amz-sdk-invocation-id': b'641c6ce8-1db7-46f2-9f19-70a44242c9dd', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,608 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,614 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/13/13c6e95c06ae06f126f5e940d6d88c2fec0da715c80878ad225c76ad48d0a31e HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,614 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqsh-cwu86p-l5g', 'x-amz-id-2': 'mhkchqsh-cwu86p-l5g', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,614 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,615 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,615 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,615 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,615 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,616 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqsh-cwu86p-l5g', 'HostId': 'mhkchqsh-cwu86p-l5g', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqsh-cwu86p-l5g', 'x-amz-id-2': 'mhkchqsh-cwu86p-l5g', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,616 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,617 [75] [DEBUG] [data.model.storage] Removing sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,617 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,618 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,618 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531'} namespacegcworker stdout | 2025-11-04 09:06:56,618 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,618 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,618 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,619 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | da63ce249a36a4143f7ced5ea3444ca8a7d9c88318d45b18a20c534e161cd405 namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 15d30348cfab98f09587b000aecca3ad693b97a32bc02e43dad15f421d537f13 namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=15d30348cfab98f09587b000aecca3ad693b97a32bc02e43dad15f421d537f13', 'amz-sdk-invocation-id': b'b12ac66a-3451-43fd-872f-bc43f409efab', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,620 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,624 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,624 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqst-d40o7h-9xf', 'x-amz-id-2': 'mhkchqst-d40o7h-9xf', 'ETag': '"89f57b307afc2e2295d31775273e0d3b-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:29 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,624 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqst-d40o7h-9xf', 'HostId': 'mhkchqst-d40o7h-9xf', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqst-d40o7h-9xf', 'x-amz-id-2': 'mhkchqst-d40o7h-9xf', 'etag': '"89f57b307afc2e2295d31775273e0d3b-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:29 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 29, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"89f57b307afc2e2295d31775273e0d3b-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531'} namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,625 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,626 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 41f948bc1db5bdf24571e945d050382e3a3f90896a419739a22c9e0a2e3043d9 namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | dbf8c1c529b3f9394957b872f13eb886f6ba61be39bf8accecac075df1217825 namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,627 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,628 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=dbf8c1c529b3f9394957b872f13eb886f6ba61be39bf8accecac075df1217825', 'amz-sdk-invocation-id': b'8a299913-5e7b-45c6-a345-c054f1721247', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,628 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,634 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/5a/5a7083e09b82b64137036dbd0e005d86eada32fb67816995c69786552f0c4531 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,634 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqt1-d8hcqc-161a', 'x-amz-id-2': 'mhkchqt1-d8hcqc-161a', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,634 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqt1-d8hcqc-161a', 'HostId': 'mhkchqt1-d8hcqc-161a', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqt1-d8hcqc-161a', 'x-amz-id-2': 'mhkchqt1-d8hcqc-161a', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,635 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,636 [75] [DEBUG] [data.model.storage] Removing sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,636 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,637 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2'} namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,638 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f365998a1c4ce1f15a9fb8fb29907e245dc375abb4b9375f9a89b34800bc3b7e namespacegcworker stdout | 2025-11-04 09:06:56,639 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | c9969e796b1ad029aa215c0b59e63f6bbe34e9c389f5d56a552430be2e455a42 namespacegcworker stdout | 2025-11-04 09:06:56,640 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,640 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,640 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,640 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c9969e796b1ad029aa215c0b59e63f6bbe34e9c389f5d56a552430be2e455a42', 'amz-sdk-invocation-id': b'3efae9e1-8d54-499e-ba87-557b9bc6f2a9', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,640 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtd-dftgwv-105h', 'x-amz-id-2': 'mhkchqtd-dftgwv-105h', 'ETag': '"c5c90e1d6112be0d9ed4bc75dd9fc727-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:25 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,644 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqtd-dftgwv-105h', 'HostId': 'mhkchqtd-dftgwv-105h', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtd-dftgwv-105h', 'x-amz-id-2': 'mhkchqtd-dftgwv-105h', 'etag': '"c5c90e1d6112be0d9ed4bc75dd9fc727-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:25 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 25, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"c5c90e1d6112be0d9ed4bc75dd9fc727-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2'} namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,645 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f30a3009259b466634398983522053a1c8b096e49b5966eb343f380e64758cc0 namespacegcworker stdout | 2025-11-04 09:06:56,646 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | e8e07f561d91c0d4506103e7aa3ab95426ebb9c83907762dcd13e23663c3ded8 namespacegcworker stdout | 2025-11-04 09:06:56,647 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,647 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,647 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,647 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e8e07f561d91c0d4506103e7aa3ab95426ebb9c83907762dcd13e23663c3ded8', 'amz-sdk-invocation-id': b'd27c81b0-eca7-45b6-a6c1-4f06ba70e6f2', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,647 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/1d/1d62db0f76abe77b24ea3b44b8a8163ae9c7b677df7da02e06c56401c952fda2 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtk-djzxoj-en9', 'x-amz-id-2': 'mhkchqtk-djzxoj-en9', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,654 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqtk-djzxoj-en9', 'HostId': 'mhkchqtk-djzxoj-en9', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtk-djzxoj-en9', 'x-amz-id-2': 'mhkchqtk-djzxoj-en9', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,655 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,655 [75] [DEBUG] [data.model.storage] Removing sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,655 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,656 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,656 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719'} namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,657 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 68dab767d53a47cb8467eaa03567828d7ecb6d94fefcb237e58816c4afd9e166 namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 50f464c75eaa03a7dd5a801c9fa6ee72d3b3b23e6bc607cdc4de8b5d389c0ac3 namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,658 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,659 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=50f464c75eaa03a7dd5a801c9fa6ee72d3b3b23e6bc607cdc4de8b5d389c0ac3', 'amz-sdk-invocation-id': b'38fe1788-3509-4f8c-934a-501b59a5a96f', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,659 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,662 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,662 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtw-dqyf9w-d7i', 'x-amz-id-2': 'mhkchqtw-dqyf9w-d7i', 'ETag': '"3177686b24ad06e2bc7bee556076a7bc-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:23 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '593', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqtw-dqyf9w-d7i', 'HostId': 'mhkchqtw-dqyf9w-d7i', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqtw-dqyf9w-d7i', 'x-amz-id-2': 'mhkchqtw-dqyf9w-d7i', 'etag': '"3177686b24ad06e2bc7bee556076a7bc-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:23 GMT', 'content-type': 'application/octet-stream', 'content-length': '593', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 23, tzinfo=tzutc()), 'ContentLength': 593, 'ETag': '"3177686b24ad06e2bc7bee556076a7bc-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,663 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719'} namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,664 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 02e170ce944677cb7c8e1ecaf3affd2f3c3293dbf4b1cc8bb4695f2c24d49cd2 namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | c0f695aa5cd431261bfcfe869c4c0d6016ffbcab9dd68cedb9b910c3a1a8d6ed namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,665 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c0f695aa5cd431261bfcfe869c4c0d6016ffbcab9dd68cedb9b910c3a1a8d6ed', 'amz-sdk-invocation-id': b'306fce30-fcea-46de-86e4-6dbe5a2ae905', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,666 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,672 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/8c/8c5706b427ad26d9b8fea9d022679a71bea070381754866e99b479f9e2465719 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqu3-duzh4b-27i', 'x-amz-id-2': 'mhkchqu3-duzh4b-27i', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,673 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqu3-duzh4b-27i', 'HostId': 'mhkchqu3-duzh4b-27i', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqu3-duzh4b-27i', 'x-amz-id-2': 'mhkchqu3-duzh4b-27i', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,674 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,674 [75] [DEBUG] [data.model.storage] Removing sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,674 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91'} namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,675 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 45e79643fbf33552b5b569c776c0fdc8670451f4e6ec7bc1f87ff44e16d60ebe namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 371101b14b02e48dfde03e25d440826be8429c2c755d814cc641f4466565eea5 namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,676 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,677 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=371101b14b02e48dfde03e25d440826be8429c2c755d814cc641f4466565eea5', 'amz-sdk-invocation-id': b'ae8d0aa4-7947-41fb-b3c1-8f8e4d0a0cb1', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,677 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,680 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,680 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchque-e1icm1-478', 'x-amz-id-2': 'mhkchque-e1icm1-478', 'ETag': '"2550f9976baadb225f6eca9b03ce8364-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:28 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '81012', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,680 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchque-e1icm1-478', 'HostId': 'mhkchque-e1icm1-478', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchque-e1icm1-478', 'x-amz-id-2': 'mhkchque-e1icm1-478', 'etag': '"2550f9976baadb225f6eca9b03ce8364-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:28 GMT', 'content-type': 'application/octet-stream', 'content-length': '81012', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 28, tzinfo=tzutc()), 'ContentLength': 81012, 'ETag': '"2550f9976baadb225f6eca9b03ce8364-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91'} namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,681 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 2d2206937309f34f89424b4cd949393c093a45f2f0df5ff4bb31d29c94831050 namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | b0a970fe0d4be06f7fac41b8ba76c05d3704c5f4ed0d29144783fcb14c3506fa namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,682 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=b0a970fe0d4be06f7fac41b8ba76c05d3704c5f4ed0d29144783fcb14c3506fa', 'amz-sdk-invocation-id': b'170ef634-05c2-4a38-8081-831f10cd9388', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,683 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,689 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/f7/f75496497526adcf2cbb76a7252b1953535592235a0d25f39a3626bed621cd91 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,689 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchquk-e5gr5a-1dpn', 'x-amz-id-2': 'mhkchquk-e5gr5a-1dpn', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,689 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchquk-e5gr5a-1dpn', 'HostId': 'mhkchquk-e5gr5a-1dpn', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchquk-e5gr5a-1dpn', 'x-amz-id-2': 'mhkchquk-e5gr5a-1dpn', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,690 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [data.model.storage] Removing sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc from local_us namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc'} namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,691 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,692 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 5fc7d4c25bd32fa4dc3326d20bac46a165691bbd9fe3c46116f8df032697b520 namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 52bdaa9138595d2ff1d73d6770450a51aa2bd8cd8cde4a73804affce04d06824 namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,693 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=52bdaa9138595d2ff1d73d6770450a51aa2bd8cd8cde4a73804affce04d06824', 'amz-sdk-invocation-id': b'b143eb4e-2e1b-4b18-97a7-be4b6e0c7263', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,694 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,698 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,698 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchquv-ebq8jc-jr7', 'x-amz-id-2': 'mhkchquv-ebq8jc-jr7', 'ETag': '"727b7c27c5d93939b3a85955e4edb460-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:27 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '597', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,698 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchquv-ebq8jc-jr7', 'HostId': 'mhkchquv-ebq8jc-jr7', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchquv-ebq8jc-jr7', 'x-amz-id-2': 'mhkchquv-ebq8jc-jr7', 'etag': '"727b7c27c5d93939b3a85955e4edb460-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:27 GMT', 'content-type': 'application/octet-stream', 'content-length': '597', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 27, tzinfo=tzutc()), 'ContentLength': 597, 'ETag': '"727b7c27c5d93939b3a85955e4edb460-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc'} namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,699 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,700 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 5076c2393d1f390a9ae84b229c0d52e212d09f2fc34c3c73ce8c46a130610341 namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | fa7a6005ea11c38063f18050c1547da70a074309ce3d802bf0650fe0386d688d namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=fa7a6005ea11c38063f18050c1547da70a074309ce3d802bf0650fe0386d688d', 'amz-sdk-invocation-id': b'f60102ba-c691-4dc9-bd4b-1f1a0162c843', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,701 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,708 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/17/171e65262c8018f8f09a360006af2acbb2c0c494cb2be933025b466ea3d672dc HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqv3-egbtqu-aub', 'x-amz-id-2': 'mhkchqv3-egbtqu-aub', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqv3-egbtqu-aub', 'HostId': 'mhkchqv3-egbtqu-aub', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqv3-egbtqu-aub', 'x-amz-id-2': 'mhkchqv3-egbtqu-aub', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,709 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,710 [75] [DEBUG] [data.model.storage] Removing sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,710 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,711 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,711 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39'} namespacegcworker stdout | 2025-11-04 09:06:56,711 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,711 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,712 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6bccc625c816accbf298091745c828b86e25fde2e5c6c485b1a1673c12b53853 namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 974557cc3865ed097092d1c527a92670af55605c52d11e1a87d6cdd0a264eae6 namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,713 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,714 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=974557cc3865ed097092d1c527a92670af55605c52d11e1a87d6cdd0a264eae6', 'amz-sdk-invocation-id': b'84c871d1-ba74-4df1-9c2f-49af8d7b8b64', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,714 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,718 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,718 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvf-enyhye-15xa', 'x-amz-id-2': 'mhkchqvf-enyhye-15xa', 'ETag': '"c91b0999fc8f0675a12fe78b393e9ebd-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:26 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '4138069', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,718 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,718 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,718 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqvf-enyhye-15xa', 'HostId': 'mhkchqvf-enyhye-15xa', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvf-enyhye-15xa', 'x-amz-id-2': 'mhkchqvf-enyhye-15xa', 'etag': '"c91b0999fc8f0675a12fe78b393e9ebd-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:26 GMT', 'content-type': 'application/octet-stream', 'content-length': '4138069', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 26, tzinfo=tzutc()), 'ContentLength': 4138069, 'ETag': '"c91b0999fc8f0675a12fe78b393e9ebd-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39'} namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,719 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,720 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f6b2b6ee389a9c6899d6acaa2d1d9b066e405ca7140ed4b63548a668437f3319 namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 47911fb448af2a77b1aaf825f219d61800d566101e8bb66aa638a05308f2072c namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,721 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=47911fb448af2a77b1aaf825f219d61800d566101e8bb66aa638a05308f2072c', 'amz-sdk-invocation-id': b'875c6132-5ab4-426a-a1b0-19842baf7ad0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,722 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,727 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6b/6b59a28fa20117e6048ad0616b8d8c901877ef15ff4c7f18db04e4f01f43bc39 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvn-esan9z-17im', 'x-amz-id-2': 'mhkchqvn-esan9z-17im', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqvn-esan9z-17im', 'HostId': 'mhkchqvn-esan9z-17im', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvn-esan9z-17im', 'x-amz-id-2': 'mhkchqvn-esan9z-17im', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,728 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,729 [75] [DEBUG] [data.model.storage] Removing sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af from local_us namespacegcworker stdout | 2025-11-04 09:06:56,729 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,729 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af'} namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,730 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 002029759ab705d3757c6c4b3cefd0dbc5313edd7662ae1f7a1f7ddb4e741b4f namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5769f01456b628e460b2766c0a16d7e9df660084f2af1698c3564e171eafa64c namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,731 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5769f01456b628e460b2766c0a16d7e9df660084f2af1698c3564e171eafa64c', 'amz-sdk-invocation-id': b'f5fc1e0a-e055-4930-a6b2-de040f8d73d1', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,732 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvx-eymyoy-l47', 'x-amz-id-2': 'mhkchqvx-eymyoy-l47', 'ETag': '"149f615ef815a59b7133efa023a9edec-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:28 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5749', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,736 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqvx-eymyoy-l47', 'HostId': 'mhkchqvx-eymyoy-l47', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqvx-eymyoy-l47', 'x-amz-id-2': 'mhkchqvx-eymyoy-l47', 'etag': '"149f615ef815a59b7133efa023a9edec-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:28 GMT', 'content-type': 'application/octet-stream', 'content-length': '5749', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 28, tzinfo=tzutc()), 'ContentLength': 5749, 'ETag': '"149f615ef815a59b7133efa023a9edec-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af'} namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,737 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | a204dac3acd07c94b727a097622243c7d09ee02b7a7a7d56f4bf37c7347d641b namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 89dbafc4af773c10534d68bdd2e673957a959a1e3bdbe0829311de6a9bda2e9c namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,738 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=89dbafc4af773c10534d68bdd2e673957a959a1e3bdbe0829311de6a9bda2e9c', 'amz-sdk-invocation-id': b'c498912d-8f59-4a5d-9f8f-ed822014c2aa', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,739 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,744 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/e1/e1186ba8d87ee119ae57765ad063c6f887ac49fa58828daeac8f5c397188a6af HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqw4-f2dbgi-18bm', 'x-amz-id-2': 'mhkchqw4-f2dbgi-18bm', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqw4-f2dbgi-18bm', 'HostId': 'mhkchqw4-f2dbgi-18bm', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqw4-f2dbgi-18bm', 'x-amz-id-2': 'mhkchqw4-f2dbgi-18bm', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,745 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,746 [75] [DEBUG] [data.model.storage] Removing sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,746 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,746 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4'} namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,747 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | def3f5c2c9588863c9addd57dbec10e1bee7f0f8950e6b1f3b7cba9921c61362 namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 49287792e8ee6aa833282f7b502de7aea8178948fdad3be78dbcecbfc3df269c namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=49287792e8ee6aa833282f7b502de7aea8178948fdad3be78dbcecbfc3df269c', 'amz-sdk-invocation-id': b'15372b8d-c346-476c-863c-05d788553e63', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,748 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqwd-f869bd-1859', 'x-amz-id-2': 'mhkchqwd-f869bd-1859', 'ETag': '"a4746760ead2594ca853c141c73e37ba-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:24 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5721', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqwd-f869bd-1859', 'HostId': 'mhkchqwd-f869bd-1859', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqwd-f869bd-1859', 'x-amz-id-2': 'mhkchqwd-f869bd-1859', 'etag': '"a4746760ead2594ca853c141c73e37ba-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:24 GMT', 'content-type': 'application/octet-stream', 'content-length': '5721', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 24, tzinfo=tzutc()), 'ContentLength': 5721, 'ETag': '"a4746760ead2594ca853c141c73e37ba-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4'} namespacegcworker stdout | 2025-11-04 09:06:56,752 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,753 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 31bcc350ea68dc2933290a8895a00a18735520f9e76fdbfb18d3e82125215100 namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 7b3d00ce30780b8e3a9fd9e9650801fedde11f6100affd3a5bbcfa3f14ffe0f3 namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7b3d00ce30780b8e3a9fd9e9650801fedde11f6100affd3a5bbcfa3f14ffe0f3', 'amz-sdk-invocation-id': b'ee77f2f2-c01f-4b20-bb0d-efacf6cfb501', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,754 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/91/916d490b7e69f787674396546a1fb03b562e336d19b9e76f804a03490fa4b4e4 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqwj-fbmk4r-154k', 'x-amz-id-2': 'mhkchqwj-fbmk4r-154k', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,760 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqwj-fbmk4r-154k', 'HostId': 'mhkchqwj-fbmk4r-154k', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqwj-fbmk4r-154k', 'x-amz-id-2': 'mhkchqwj-fbmk4r-154k', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,761 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,761 [75] [DEBUG] [data.model.gc] Found 9 uploaded blobs to GC under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,761 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 67 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,762 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [67]) namespacegcworker stdout | 2025-11-04 09:06:56,763 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 68 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,763 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,765 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 69 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,765 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,766 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 70 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,767 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,768 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 71 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,768 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,769 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 72 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,769 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,771 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 73 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,771 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,772 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 74 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,772 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,774 [75] [DEBUG] [data.model.gc] Deleting uploaded blob 75 under repository 8 namespacegcworker stdout | 2025-11-04 09:06:56,774 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,775 [75] [DEBUG] [data.model.storage] Garbage collecting storages from candidates: {68, 69, 70, 71, 72, 73, 74, 75, 76} namespacegcworker stdout | 2025-11-04 09:06:56,775 [75] [DEBUG] [data.model.storage] Garbage collecting storage 68 namespacegcworker stdout | 2025-11-04 09:06:56,776 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,777 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [68, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,778 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,779 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,779 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,779 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [68]) namespacegcworker stdout | 2025-11-04 09:06:56,780 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5']) namespacegcworker stdout | 2025-11-04 09:06:56,781 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5']) namespacegcworker stdout | 2025-11-04 09:06:56,782 [75] [DEBUG] [data.model.storage] Garbage collecting storage 69 namespacegcworker stdout | 2025-11-04 09:06:56,783 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,784 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [69, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,785 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,785 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,786 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,786 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [69]) namespacegcworker stdout | 2025-11-04 09:06:56,787 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2']) namespacegcworker stdout | 2025-11-04 09:06:56,787 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2']) namespacegcworker stdout | 2025-11-04 09:06:56,789 [75] [DEBUG] [data.model.storage] Garbage collecting storage 70 namespacegcworker stdout | 2025-11-04 09:06:56,789 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,790 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [70, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,791 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,792 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,793 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,793 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [70]) namespacegcworker stdout | 2025-11-04 09:06:56,794 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786']) namespacegcworker stdout | 2025-11-04 09:06:56,794 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786']) namespacegcworker stdout | 2025-11-04 09:06:56,796 [75] [DEBUG] [data.model.storage] Garbage collecting storage 71 namespacegcworker stdout | 2025-11-04 09:06:56,796 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,797 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [71, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,798 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,799 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,799 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,800 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [71]) namespacegcworker stdout | 2025-11-04 09:06:56,801 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956']) namespacegcworker stdout | 2025-11-04 09:06:56,801 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956']) namespacegcworker stdout | 2025-11-04 09:06:56,803 [75] [DEBUG] [data.model.storage] Garbage collecting storage 72 namespacegcworker stdout | 2025-11-04 09:06:56,803 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,804 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [72, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,805 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,806 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,806 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,807 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [72]) namespacegcworker stdout | 2025-11-04 09:06:56,807 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf']) namespacegcworker stdout | 2025-11-04 09:06:56,808 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf']) namespacegcworker stdout | 2025-11-04 09:06:56,810 [75] [DEBUG] [data.model.storage] Garbage collecting storage 73 namespacegcworker stdout | 2025-11-04 09:06:56,810 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,811 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [73, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,812 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,813 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,813 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,814 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [73]) namespacegcworker stdout | 2025-11-04 09:06:56,815 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28']) namespacegcworker stdout | 2025-11-04 09:06:56,816 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28']) namespacegcworker stdout | 2025-11-04 09:06:56,817 [75] [DEBUG] [data.model.storage] Garbage collecting storage 74 namespacegcworker stdout | 2025-11-04 09:06:56,818 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,819 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [74, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,819 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,820 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,820 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,821 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [74]) namespacegcworker stdout | 2025-11-04 09:06:56,822 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71']) namespacegcworker stdout | 2025-11-04 09:06:56,822 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71']) namespacegcworker stdout | 2025-11-04 09:06:56,824 [75] [DEBUG] [data.model.storage] Garbage collecting storage 75 namespacegcworker stdout | 2025-11-04 09:06:56,824 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,825 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [75, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,826 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,827 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,827 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,828 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [75]) namespacegcworker stdout | 2025-11-04 09:06:56,828 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b']) namespacegcworker stdout | 2025-11-04 09:06:56,829 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b']) namespacegcworker stdout | 2025-11-04 09:06:56,831 [75] [DEBUG] [data.model.storage] Garbage collecting storage 76 namespacegcworker stdout | 2025-11-04 09:06:56,831 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_id", "t1"."blob_id" FROM "manifestblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,832 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."blob_id" = %s) LIMIT %s OFFSET %s', [76, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:56,833 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."id" = %s)', [76]) namespacegcworker stdout | 2025-11-04 09:06:56,833 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorageplacement" WHERE ("imagestorageplacement"."storage_id" = %s)', [76]) namespacegcworker stdout | 2025-11-04 09:06:56,834 [75] [DEBUG] [peewee] ('DELETE FROM "imagestoragesignature" WHERE ("imagestoragesignature"."storage_id" = %s)', [76]) namespacegcworker stdout | 2025-11-04 09:06:56,834 [75] [DEBUG] [peewee] ('DELETE FROM "imagestorage" WHERE ("imagestorage"."id" = %s)', [76]) namespacegcworker stdout | 2025-11-04 09:06:56,835 [75] [DEBUG] [peewee] ('SELECT "t1"."content_checksum" FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" IN (%s))', ['sha256:050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950']) namespacegcworker stdout | 2025-11-04 09:06:56,836 [75] [DEBUG] [peewee] ('SELECT "t1"."digest" FROM "apprblob" AS "t1" WHERE ("t1"."digest" IN (%s))', ['sha256:050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950']) namespacegcworker stdout | 2025-11-04 09:06:56,838 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,838 [75] [DEBUG] [data.model.storage] Removing sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf from local_us namespacegcworker stdout | 2025-11-04 09:06:56,838 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf'} namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,839 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,840 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | f4e1d61ade42581b249cb766aec22286d6afe6bac169fe09268cd8fb5cf9d68e namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6dc86218e5dc8d8b820e76976ad2c27637bfbcaebb2ed8a5faf6d1e6976e5bb5 namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6dc86218e5dc8d8b820e76976ad2c27637bfbcaebb2ed8a5faf6d1e6976e5bb5', 'amz-sdk-invocation-id': b'7aa0f28a-d75b-4787-a1db-a9c456ffe1c7', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,841 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,845 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,845 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqyy-86mcc-lyl', 'x-amz-id-2': 'mhkchqyy-86mcc-lyl', 'ETag': '"3d7beaab59be96e8f3bb4d932273dc17-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:34 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '3732241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,845 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqyy-86mcc-lyl', 'HostId': 'mhkchqyy-86mcc-lyl', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqyy-86mcc-lyl', 'x-amz-id-2': 'mhkchqyy-86mcc-lyl', 'etag': '"3d7beaab59be96e8f3bb4d932273dc17-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:34 GMT', 'content-type': 'application/octet-stream', 'content-length': '3732241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 34, tzinfo=tzutc()), 'ContentLength': 3732241, 'ETag': '"3d7beaab59be96e8f3bb4d932273dc17-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf'} namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,846 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 28a86c2a6e28e207630ff0fafdfb52124e5c18589f8c90bedf7664a5d9d9a979 namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 529b140f891288b14c192238cfe2e74e80c8136161bfed2d77baf6451578fd70 namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,847 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=529b140f891288b14c192238cfe2e74e80c8136161bfed2d77baf6451578fd70', 'amz-sdk-invocation-id': b'ecf2d106-ce19-45e2-9328-70b7e4d3f8db', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,848 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/85/85a0f69f026b4a01420490809bed190217e05518f7b718c0bbc1ad4871e0dedf HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqz4-buxl5-8h6', 'x-amz-id-2': 'mhkchqz4-buxl5-8h6', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqz4-buxl5-8h6', 'HostId': 'mhkchqz4-buxl5-8h6', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqz4-buxl5-8h6', 'x-amz-id-2': 'mhkchqz4-buxl5-8h6', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,854 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,855 [75] [DEBUG] [data.model.storage] Removing sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,855 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,855 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5'} namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,856 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | dd08c5efdc402887b39e64459600c81becda9ab383040e2ecd2a78a3692c94ab namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5a3a3facf9a3b551318114f2484aaa008ad4a31b059b0858100d3d30701728a7 namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,857 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5a3a3facf9a3b551318114f2484aaa008ad4a31b059b0858100d3d30701728a7', 'amz-sdk-invocation-id': b'5c1ce216-5679-4b94-8ccb-7a81b7ca2029', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,858 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,861 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,861 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzf-i1zrq-10y8', 'x-amz-id-2': 'mhkchqzf-i1zrq-10y8', 'ETag': '"77fa9c8062fde936b3fdb16c45153182-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:30 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '576', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,861 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqzf-i1zrq-10y8', 'HostId': 'mhkchqzf-i1zrq-10y8', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzf-i1zrq-10y8', 'x-amz-id-2': 'mhkchqzf-i1zrq-10y8', 'etag': '"77fa9c8062fde936b3fdb16c45153182-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:30 GMT', 'content-type': 'application/octet-stream', 'content-length': '576', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 30, tzinfo=tzutc()), 'ContentLength': 576, 'ETag': '"77fa9c8062fde936b3fdb16c45153182-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5'} namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,862 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 1e87f614ff8f90028fa810879d6e745b69c29bd354000adc12bb517be77428e2 namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | a2ea86f67065cabff5bc6a0fdef4ac041630c5593e56eb20a48f5b9b31932c7f namespacegcworker stdout | 2025-11-04 09:06:56,863 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,864 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,864 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,864 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a2ea86f67065cabff5bc6a0fdef4ac041630c5593e56eb20a48f5b9b31932c7f', 'amz-sdk-invocation-id': b'd83aac24-db54-4ab8-872c-20e76854aac3', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,864 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/6a/6ae11c41c28774b9a91cd9b478a7347b8dad420383bbd4249d446363262947f5 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzl-lm23c-yy8', 'x-amz-id-2': 'mhkchqzl-lm23c-yy8', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,870 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqzl-lm23c-yy8', 'HostId': 'mhkchqzl-lm23c-yy8', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzl-lm23c-yy8', 'x-amz-id-2': 'mhkchqzl-lm23c-yy8', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,871 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,871 [75] [DEBUG] [data.model.storage] Removing sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b from local_us namespacegcworker stdout | 2025-11-04 09:06:56,871 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b'} namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,873 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 67d4f6fd5a78b632c8537d6cb2a026ac2c41ebec3ec835474e189c542bf59edf namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 7a56fbbccad4039fd30899b5cea5e6adc4c9d77806ff098add25d8b4302dcb9d namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a56fbbccad4039fd30899b5cea5e6adc4c9d77806ff098add25d8b4302dcb9d', 'amz-sdk-invocation-id': b'53c99ce2-d338-4063-8efc-d94d3e46dda3', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,874 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,878 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzv-rsgke-175c', 'x-amz-id-2': 'mhkchqzv-rsgke-175c', 'ETag': '"77144b95d5b53a621b3ce90640d03894-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:36 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '79039', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchqzv-rsgke-175c', 'HostId': 'mhkchqzv-rsgke-175c', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchqzv-rsgke-175c', 'x-amz-id-2': 'mhkchqzv-rsgke-175c', 'etag': '"77144b95d5b53a621b3ce90640d03894-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:36 GMT', 'content-type': 'application/octet-stream', 'content-length': '79039', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 36, tzinfo=tzutc()), 'ContentLength': 79039, 'ETag': '"77144b95d5b53a621b3ce90640d03894-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b'} namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,879 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b35890a89e25a5ae6a2c6fd76863649c947e5ccb033227f1c81e5d88b0362318 namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 7c91dd5666488f8ca2e8ba510bc91b7f3405d36329b6103200fba79e9880401f namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,880 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,881 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,881 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7c91dd5666488f8ca2e8ba510bc91b7f3405d36329b6103200fba79e9880401f', 'amz-sdk-invocation-id': b'974a6645-19c1-4bbb-9da9-dfa8c89b921a', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,881 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/45/458183e7c96429f0bdac206f68a19394640f69d97639921519d9890f5b49b30b HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr02-vmtss-wkc', 'x-amz-id-2': 'mhkchr02-vmtss-wkc', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,888 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr02-vmtss-wkc', 'HostId': 'mhkchr02-vmtss-wkc', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr02-vmtss-wkc', 'x-amz-id-2': 'mhkchr02-vmtss-wkc', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,889 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,889 [75] [DEBUG] [data.model.storage] Removing sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,889 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28'} namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,890 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0e4684b2112f184d11d189982cdef1504880d7d7091fa4e88a4ef82abed4929b namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | d540816a8509b8a318eadc5277d30ae4d7dc8da0adbd11a59e9c4eaedd950764 namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=d540816a8509b8a318eadc5277d30ae4d7dc8da0adbd11a59e9c4eaedd950764', 'amz-sdk-invocation-id': b'c7f3ccc1-2fca-4af1-a788-3c3dc5d417ee', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,891 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0c-11wo3e-11yi', 'x-amz-id-2': 'mhkchr0c-11wo3e-11yi', 'ETag': '"451a7f1733b159a6e24777fca2c2989e-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:35 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '584', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,895 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr0c-11wo3e-11yi', 'HostId': 'mhkchr0c-11wo3e-11yi', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0c-11wo3e-11yi', 'x-amz-id-2': 'mhkchr0c-11wo3e-11yi', 'etag': '"451a7f1733b159a6e24777fca2c2989e-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:35 GMT', 'content-type': 'application/octet-stream', 'content-length': '584', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 35, tzinfo=tzutc()), 'ContentLength': 584, 'ETag': '"451a7f1733b159a6e24777fca2c2989e-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28'} namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,896 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,897 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,897 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,897 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,897 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | edec97329cd294ac81c12fa285aedabe21b6be989710b445209038ba313fb325 namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 6184324d8ef9b3d39308f0fc72d6d128286fc0e1dd7f86850457fc16c2b426c1 namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6184324d8ef9b3d39308f0fc72d6d128286fc0e1dd7f86850457fc16c2b426c1', 'amz-sdk-invocation-id': b'a8206dd4-5ed0-47b2-ad88-0060a87652a2', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,898 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a3/a340bd06c407709c5ce3265453a4a03bac1dbeaf9a20406d89462361d85f0b28 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0j-15z7z0-g7q', 'x-amz-id-2': 'mhkchr0j-15z7z0-g7q', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr0j-15z7z0-g7q', 'HostId': 'mhkchr0j-15z7z0-g7q', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0j-15z7z0-g7q', 'x-amz-id-2': 'mhkchr0j-15z7z0-g7q', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,904 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,905 [75] [DEBUG] [data.model.storage] Removing sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,905 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,905 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950'} namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,906 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | efb9d5f7ffa99eb568c7769ad0b6dcad236d143d7449ec4b0e01b586697ea4d0 namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | a846f37ca815a800c12c6298105465065d0b227508e921045e22a13c4715002b namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,907 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a846f37ca815a800c12c6298105465065d0b227508e921045e22a13c4715002b', 'amz-sdk-invocation-id': b'18ae8747-8568-4d73-9f0c-0931ab56d38e', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,908 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,911 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0t-1btmkd-pmp', 'x-amz-id-2': 'mhkchr0t-1btmkd-pmp', 'ETag': '"e63f02f4d6bfdb52fa4bc78911f34fbd-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:37 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr0t-1btmkd-pmp', 'HostId': 'mhkchr0t-1btmkd-pmp', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0t-1btmkd-pmp', 'x-amz-id-2': 'mhkchr0t-1btmkd-pmp', 'etag': '"e63f02f4d6bfdb52fa4bc78911f34fbd-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:37 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 37, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"e63f02f4d6bfdb52fa4bc78911f34fbd-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950'} namespacegcworker stdout | 2025-11-04 09:06:56,912 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,913 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 8463ac69fb95464ac8dc8e99f10c475af468ed6e6cfca5d529014fa65be110ec namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 9612669382be0139ce9aa3dc6a066fd5a54ef4f98eb5758fb740648cb758189f namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9612669382be0139ce9aa3dc6a066fd5a54ef4f98eb5758fb740648cb758189f', 'amz-sdk-invocation-id': b'a273870b-70be-4a40-ba18-00891fc044c0', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,914 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/05/050d2042e170096c4df38f07c2273bbe9ea58c60f8472f9f1b2afaf6da346950 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0z-1fip3s-2kd', 'x-amz-id-2': 'mhkchr0z-1fip3s-2kd', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr0z-1fip3s-2kd', 'HostId': 'mhkchr0z-1fip3s-2kd', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr0z-1fip3s-2kd', 'x-amz-id-2': 'mhkchr0z-1fip3s-2kd', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,920 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,921 [75] [DEBUG] [data.model.storage] Removing sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,921 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,922 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,922 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956'} namespacegcworker stdout | 2025-11-04 09:06:56,922 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,922 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,922 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,923 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,924 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,924 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,924 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,924 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | d452f3d4aecc48d91a442327bab9dc03e5441612ece30cc7002efbdddab59981 namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | f28c43b1286b2e952082b64b071b0fbe84419b4ba858e96acd8015a4369987b9 namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f28c43b1286b2e952082b64b071b0fbe84419b4ba858e96acd8015a4369987b9', 'amz-sdk-invocation-id': b'823f8486-3844-4e8a-994e-01033650a026', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,925 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,929 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,929 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1a-1m6qda-15yh', 'x-amz-id-2': 'mhkchr1a-1m6qda-15yh', 'ETag': '"3550b125b9dbfd474531147f3a57b3e2-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '241', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,929 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,930 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,930 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,930 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,930 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr1a-1m6qda-15yh', 'HostId': 'mhkchr1a-1m6qda-15yh', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1a-1m6qda-15yh', 'x-amz-id-2': 'mhkchr1a-1m6qda-15yh', 'etag': '"3550b125b9dbfd474531147f3a57b3e2-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'content-type': 'application/octet-stream', 'content-length': '241', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 32, tzinfo=tzutc()), 'ContentLength': 241, 'ETag': '"3550b125b9dbfd474531147f3a57b3e2-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956'} namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,931 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e2d1d74893bfee05186f57e696889742d954e67055a90a9b10328a821e7003ae namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 1742081e791857acb0fa7455ee7c74bae1a840aebbe61a4b90a3a6018d4a9b57 namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,932 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1742081e791857acb0fa7455ee7c74bae1a840aebbe61a4b90a3a6018d4a9b57', 'amz-sdk-invocation-id': b'f25c4bec-517f-4162-9ae7-f84e88379da2', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,933 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/2a/2ab5df78ee2e1734968f9d098a965a5119c4e66c2f9afe5cedcc1f43a6bfa956 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1i-1qukil-e0w', 'x-amz-id-2': 'mhkchr1i-1qukil-e0w', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,939 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr1i-1qukil-e0w', 'HostId': 'mhkchr1i-1qukil-e0w', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1i-1qukil-e0w', 'x-amz-id-2': 'mhkchr1i-1qukil-e0w', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,940 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,940 [75] [DEBUG] [data.model.storage] Removing sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,940 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2'} namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,941 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,942 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | d4255b585d7bfafac3826e316793aa9cfe35cd65cef2b7cd9fcf7bfd5e90f07a namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5ae5251ab9d5fd06c07d39fd44da024fd8874e250320ef1844d33a132f2f0f89 namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5ae5251ab9d5fd06c07d39fd44da024fd8874e250320ef1844d33a132f2f0f89', 'amz-sdk-invocation-id': b'6d5e529f-fd64-4b9f-9105-44e5bf50a95b', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,943 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,947 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,947 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1s-1wyop4-171z', 'x-amz-id-2': 'mhkchr1s-1wyop4-171z', 'ETag': '"343d212e238e5264200b4c5f96ee17ff-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5643', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr1s-1wyop4-171z', 'HostId': 'mhkchr1s-1wyop4-171z', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1s-1wyop4-171z', 'x-amz-id-2': 'mhkchr1s-1wyop4-171z', 'etag': '"343d212e238e5264200b4c5f96ee17ff-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'content-type': 'application/octet-stream', 'content-length': '5643', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 32, tzinfo=tzutc()), 'ContentLength': 5643, 'ETag': '"343d212e238e5264200b4c5f96ee17ff-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2'} namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,948 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | b8050b204778c6d9fe02b1bed79ae72d823dc6662bccf80734ff880e89073a29 namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | b8f7ebb23dfdcc347d87f8f16e3c2b9abe6a48d0a33183c59e3d7e5891e42462 namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,949 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=b8f7ebb23dfdcc347d87f8f16e3c2b9abe6a48d0a33183c59e3d7e5891e42462', 'amz-sdk-invocation-id': b'ab4f37a2-e8bb-4c04-abea-077637be75c7', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,950 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/0d/0df9f945c965a46cc0bb181d38850468d44fca9a1244eb3dbf611dad49eec2a2 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1y-20kj81-f4e', 'x-amz-id-2': 'mhkchr1y-20kj81-f4e', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,956 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr1y-20kj81-f4e', 'HostId': 'mhkchr1y-20kj81-f4e', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr1y-20kj81-f4e', 'x-amz-id-2': 'mhkchr1y-20kj81-f4e', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,957 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,957 [75] [DEBUG] [data.model.storage] Removing sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,957 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71'} namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,958 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | cceecc424850cca4667e4665d80e2ccfaf7852a3f09b3f5f293567b5b9bb4993 namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 5a0fd6074d419ae96a83b59341fffe05f04f1566971152333040e2bb782510ea namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,959 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5a0fd6074d419ae96a83b59341fffe05f04f1566971152333040e2bb782510ea', 'amz-sdk-invocation-id': b'bcb7fe18-cea1-4a13-b7c2-ba87fd01fd31', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,960 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr29-26k2ez-mrt', 'x-amz-id-2': 'mhkchr29-26k2ez-mrt', 'ETag': '"aa2bd1693d816970a25e5a6a8c82e9f4-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:36 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '5718', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,963 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr29-26k2ez-mrt', 'HostId': 'mhkchr29-26k2ez-mrt', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr29-26k2ez-mrt', 'x-amz-id-2': 'mhkchr29-26k2ez-mrt', 'etag': '"aa2bd1693d816970a25e5a6a8c82e9f4-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:36 GMT', 'content-type': 'application/octet-stream', 'content-length': '5718', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 36, tzinfo=tzutc()), 'ContentLength': 5718, 'ETag': '"aa2bd1693d816970a25e5a6a8c82e9f4-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71'} namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,964 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | e87fd122f98e9e6516f80a55ce95a29c1ad044db66953f45a3f4d19518a875e2 namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 407703cbacd18771c74ade20338af888fb8e6a4092291bfc92afc999a9e3f872 namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=407703cbacd18771c74ade20338af888fb8e6a4092291bfc92afc999a9e3f872', 'amz-sdk-invocation-id': b'0c962c56-ae50-42ea-8def-7dad2dd8643a', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,965 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,971 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/9b/9bfbcacb65cf66c8b5ee52caafc060575589bac45af4a49499d9aa15b057ea71 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2e-29rfne-3kp', 'x-amz-id-2': 'mhkchr2e-29rfne-3kp', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr2e-29rfne-3kp', 'HostId': 'mhkchr2e-29rfne-3kp', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2e-29rfne-3kp', 'x-amz-id-2': 'mhkchr2e-29rfne-3kp', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,972 [75] [DEBUG] [peewee] ('SELECT 1 FROM "imagestorage" AS "t1" WHERE ("t1"."content_checksum" = %s) LIMIT %s', ['sha256:7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 1]) namespacegcworker stdout | 2025-11-04 09:06:56,973 [75] [DEBUG] [data.model.storage] Removing sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786 from local_us namespacegcworker stdout | 2025-11-04 09:06:56,973 [75] [DEBUG] [boto3.resources.factory] Loading s3:Object namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Object: calling handler ._handler at 0x7fbe73e4ade0> namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [boto3.resources.action] Calling s3:head_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786'} namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-call.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadObject) with params: {'url_path': '/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,974 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | HEAD namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 6a69f3ea755a5a9b6aefc7a41505846d8e6d01043ecae5ef42f83e3b8cf458c1 namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 98bcf02d3e1747d39fa8d84c97116d4cd689fd8b0b2ede918e3cd8f0d9d3cc60 namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.hooks] Event request-created.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.hooks] Event before-send.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=98bcf02d3e1747d39fa8d84c97116d4cd689fd8b0b2ede918e3cd8f0d9d3cc60', 'amz-sdk-invocation-id': b'f4edaffa-acac-4109-84f4-afeee6b084f1', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,975 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786 HTTP/1.1" 200 0 namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2o-2ftvpt-mff', 'x-amz-id-2': 'mhkchr2o-2ftvpt-mff', 'ETag': '"6d08d79ff5b12e4e2bba7527bc9c63f2-1"', 'Last-Modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'Content-Type': 'application/octet-stream', 'Content-Length': '80931', 'Accept-Ranges': 'bytes', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event after-call.s3.HeadObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr2o-2ftvpt-mff', 'HostId': 'mhkchr2o-2ftvpt-mff', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2o-2ftvpt-mff', 'x-amz-id-2': 'mhkchr2o-2ftvpt-mff', 'etag': '"6d08d79ff5b12e4e2bba7527bc9c63f2-1"', 'last-modified': 'Tue, 04 Nov 2025 08:26:32 GMT', 'content-type': 'application/octet-stream', 'content-length': '80931', 'accept-ranges': 'bytes', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2025, 11, 4, 8, 26, 32, tzinfo=tzutc()), 'ContentLength': 80931, 'ETag': '"6d08d79ff5b12e4e2bba7527bc9c63f2-1"', 'ContentType': 'application/octet-stream', 'Metadata': {}} namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [boto3.resources.action] Calling s3:delete_object with {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786'} namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,979 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-call.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=DeleteObject) with params: {'url_path': '/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'query_string': {}, 'method': 'DELETE', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Key': 'datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event choose-signer.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.hooks] Event before-sign.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.auth] Calculating signature using v4 auth. namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.auth] CanonicalRequest: namespacegcworker stdout | DELETE namespacegcworker stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786 namespacegcworker stdout | host:s3.openshift-storage.svc.cluster.local namespacegcworker stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | x-amz-date:20251104T090656Z namespacegcworker stdout | host;x-amz-content-sha256;x-amz-date namespacegcworker stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.auth] StringToSign: namespacegcworker stdout | AWS4-HMAC-SHA256 namespacegcworker stdout | 20251104T090656Z namespacegcworker stdout | 20251104/us-east-1/s3/aws4_request namespacegcworker stdout | 0b902cbc5e946cf0be1ce7f3e0dafa77852d770997479fdfc3b8d3632c8af744 namespacegcworker stdout | 2025-11-04 09:06:56,980 [75] [DEBUG] [botocore.auth] Signature: namespacegcworker stdout | 2b78066037d7bd1049abfd9bbd33d4a325190af0920f6613370ebc07b33b54b4 namespacegcworker stdout | 2025-11-04 09:06:56,981 [75] [DEBUG] [botocore.hooks] Event request-created.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,981 [75] [DEBUG] [botocore.endpoint] Sending http request: namespacegcworker stdout | 2025-11-04 09:06:56,981 [75] [DEBUG] [botocore.hooks] Event before-send.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,981 [75] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61 Resource', 'X-Amz-Date': b'20251104T090656Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2b78066037d7bd1049abfd9bbd33d4a325190af0920f6613370ebc07b33b54b4', 'amz-sdk-invocation-id': b'f070e161-fbe8-4aaa-9277-de72aeffbc98', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} namespacegcworker stdout | 2025-11-04 09:06:56,981 [75] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "DELETE /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/72/7209d55fc81fa572dae1ee6c83078b80850f9a0d8c12877527f93ff752277786 HTTP/1.1" 204 0 namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2v-2jobnx-zx6', 'x-amz-id-2': 'mhkchr2v-2jobnx-zx6', 'Date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.parsers] Response body: namespacegcworker stdout | b'' namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.retryhandler] No retry needed. namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.hooks] Event needs-retry.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [botocore.hooks] Event after-call.s3.DeleteObject: calling handler > namespacegcworker stdout | 2025-11-04 09:06:56,988 [75] [DEBUG] [boto3.resources.action] Response: {'ResponseMetadata': {'RequestId': 'mhkchr2v-2jobnx-zx6', 'HostId': 'mhkchr2v-2jobnx-zx6', 'HTTPStatusCode': 204, 'HTTPHeaders': {'server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkchr2v-2jobnx-zx6', 'x-amz-id-2': 'mhkchr2v-2jobnx-zx6', 'date': 'Tue, 04 Nov 2025 09:06:56 GMT', 'connection': 'keep-alive', 'keep-alive': 'timeout=5'}, 'RetryAttempts': 0}} namespacegcworker stdout | 2025-11-04 09:06:56,989 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,989 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."blob_id", "t1"."uploaded_at", "t1"."expires_at" FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s) LIMIT %s', [8, 10]) namespacegcworker stdout | 2025-11-04 09:06:56,990 [75] [DEBUG] [peewee] ('DELETE FROM "quotarepositorysize" WHERE ("quotarepositorysize"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,992 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "quotarepositorysize" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,992 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,993 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,994 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,995 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,996 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "apprtag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,998 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tag" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,999 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifest" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:56,999 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,000 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "uploadedblob" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,001 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestsecuritystatus" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,002 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "tagpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,003 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT 1 FROM "manifestpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)) AS "_wrapped"', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,004 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,005 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorypermission" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,006 [75] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repositorypermission" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."id" <= %s))', [8, 503]) namespacegcworker stdout | 2025-11-04 09:06:57,006 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ((("repositorypermission"."repository_id" = %s) AND ("repositorypermission"."id" >= %s)) AND ("repositorypermission"."id" <= %s))', [8, 3, 3]) namespacegcworker stdout | 2025-11-04 09:06:57,008 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorypermission" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,009 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorybuild" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,010 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorybuildtrigger" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,012 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,012 [75] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."id" <= %s))', [8, 503]) namespacegcworker stdout | 2025-11-04 09:06:57,013 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryactioncount" WHERE ((("repositoryactioncount"."repository_id" = %s) AND ("repositoryactioncount"."id" >= %s)) AND ("repositoryactioncount"."id" <= %s))', [8, 3, 3]) namespacegcworker stdout | 2025-11-04 09:06:57,015 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryactioncount" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,015 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "star" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,016 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "accesstoken" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,017 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositorynotification" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,018 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "blobupload" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,019 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,020 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repositoryauthorizedemail" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,021 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "tagpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,022 [75] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifestpullstatistics" AS "t1" WHERE ("t1"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,022 [75] [DEBUG] [peewee] ('DELETE FROM "deletedrepository" WHERE ("deletedrepository"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,023 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [8, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:57,024 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuild" WHERE ("repositorybuild"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,024 [75] [DEBUG] [peewee] ('DELETE FROM "tag" WHERE ("tag"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,025 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,026 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE ("repomirrorconfig"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,026 [75] [DEBUG] [peewee] ('DELETE FROM "manifestsecuritystatus" WHERE ("manifestsecuritystatus"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,027 [75] [DEBUG] [peewee] ('DELETE FROM "manifestlabel" WHERE ("manifestlabel"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,027 [75] [DEBUG] [peewee] ('DELETE FROM "manifestchild" WHERE ("manifestchild"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,028 [75] [DEBUG] [peewee] ('DELETE FROM "manifestblob" WHERE ("manifestblob"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,028 [75] [DEBUG] [peewee] ('DELETE FROM "uploadedblob" WHERE ("uploadedblob"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,028 [75] [DEBUG] [peewee] ('DELETE FROM "tagpullstatistics" WHERE ("tagpullstatistics"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,029 [75] [DEBUG] [peewee] ('DELETE FROM "star" WHERE ("star"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,029 [75] [DEBUG] [peewee] ('DELETE FROM "repositorysearchscore" WHERE ("repositorysearchscore"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,031 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ("repositorypermission"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,032 [75] [DEBUG] [peewee] ('DELETE FROM "repositorynotification" WHERE ("repositorynotification"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,032 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,033 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryauthorizedemail" WHERE ("repositoryauthorizedemail"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,033 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryactioncount" WHERE ("repositoryactioncount"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,033 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorrule" WHERE ("repomirrorrule"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,034 [75] [DEBUG] [peewee] ('DELETE FROM "quotarepositorysize" WHERE ("quotarepositorysize"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,034 [75] [DEBUG] [peewee] ('DELETE FROM "manifestpullstatistics" WHERE ("manifestpullstatistics"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,035 [75] [DEBUG] [peewee] ('DELETE FROM "manifest" WHERE ("manifest"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,035 [75] [DEBUG] [peewee] ('DELETE FROM "deletedrepository" WHERE ("deletedrepository"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,036 [75] [DEBUG] [peewee] ('DELETE FROM "blobupload" WHERE ("blobupload"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,036 [75] [DEBUG] [peewee] ('DELETE FROM "apprtag" WHERE ("apprtag"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,036 [75] [DEBUG] [peewee] ('DELETE FROM "accesstoken" WHERE ("accesstoken"."repository_id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,037 [75] [DEBUG] [peewee] ('DELETE FROM "repository" WHERE ("repository"."id" = %s)', [8]) namespacegcworker stdout | 2025-11-04 09:06:57,041 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."namespace_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,042 [75] [DEBUG] [peewee] ('DELETE FROM "autoprunetaskstatus" WHERE ("autoprunetaskstatus"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,042 [75] [DEBUG] [peewee] ('DELETE FROM "namespaceautoprunepolicy" WHERE ("namespaceautoprunepolicy"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,043 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."organization_id", "t1"."role_id", "t1"."description" FROM "team" AS "t1" WHERE ("t1"."organization_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,045 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."client_id", "t1"."secure_client_secret", "t1"."fully_migrated", "t1"."redirect_uri", "t1"."application_uri", "t1"."organization_id", "t1"."name", "t1"."description", "t1"."gravatar_email" FROM "oauthapplication" AS "t1" WHERE ("t1"."organization_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,046 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."service_id", "t1"."repository_id", "t1"."connected_user_id", "t1"."secure_auth_token", "t1"."secure_private_key", "t1"."fully_migrated", "t1"."config", "t1"."write_token_id", "t1"."pull_robot_id", "t1"."enabled", "t1"."disabled_reason_id", "t1"."disabled_datetime", "t1"."successive_failure_count", "t1"."successive_internal_error_count" FROM "repositorybuildtrigger" AS "t1" WHERE ("t1"."connected_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,046 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."limit_bytes" FROM "userorganizationquota" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['04617bc0-5074-4229-9551-bb696f9ce5fe']) namespacegcworker stdout | 2025-11-04 09:06:57,048 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed", "t2"."id", "t2"."robot_account_id", "t2"."token", "t2"."fully_migrated", "t3"."id", "t3"."robot_account_id", "t3"."description", "t3"."unstructured_json" FROM "user" AS "t1" LEFT OUTER JOIN "robotaccountmetadata" AS "t3" ON ("t3"."robot_account_id" = "t1"."id") INNER JOIN "robotaccounttoken" AS "t2" ON ("t2"."robot_account_id" = "t1"."id") WHERE (("t1"."robot" = %s) AND ("t1"."username" ILIKE %s))', [True, '04617bc0-5074-4229-9551-bb696f9ce5fe+%']) namespacegcworker stdout | 2025-11-04 09:06:57,048 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE (0 = 1)', []) namespacegcworker stdout | 2025-11-04 09:06:57,049 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed", "t2"."id", "t2"."robot_account_id", "t2"."token", "t2"."fully_migrated", "t3"."id", "t3"."robot_account_id", "t3"."description", "t3"."unstructured_json" FROM "user" AS "t1" LEFT OUTER JOIN "robotaccountmetadata" AS "t3" ON ("t3"."robot_account_id" = "t1"."id") INNER JOIN "robotaccounttoken" AS "t2" ON ("t2"."robot_account_id" = "t1"."id") WHERE (("t1"."robot" = %s) AND ("t1"."username" ILIKE %s))', [True, '04617bc0-5074-4229-9551-bb696f9ce5fe+%']) namespacegcworker stdout | 2025-11-04 09:06:57,051 [75] [DEBUG] [peewee] ('UPDATE "servicekeyapproval" SET "approver_id" = %s WHERE ("servicekeyapproval"."approver_id" = %s)', [None, 5]) namespacegcworker stdout | 2025-11-04 09:06:57,052 [75] [DEBUG] [peewee] ('DELETE FROM "federatedlogin" WHERE ("federatedlogin"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,052 [75] [DEBUG] [peewee] ('DELETE FROM "quotanamespacesize" WHERE ("quotanamespacesize"."namespace_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,053 [75] [DEBUG] [peewee] ('DELETE FROM "oauthassignedtoken" WHERE ("oauthassignedtoken"."assigned_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,055 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuild" WHERE ("repositorybuild"."pull_robot_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,056 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."pull_robot_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,056 [75] [DEBUG] [peewee] ('DELETE FROM "repositorybuildtrigger" WHERE ("repositorybuildtrigger"."connected_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,057 [75] [DEBUG] [peewee] ('DELETE FROM "repomirrorconfig" WHERE ("repomirrorconfig"."internal_robot_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,057 [75] [DEBUG] [peewee] ('DELETE FROM "teammemberinvite" WHERE ("teammemberinvite"."inviter_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,058 [75] [DEBUG] [peewee] ('DELETE FROM "teammemberinvite" WHERE ("teammemberinvite"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,058 [75] [DEBUG] [peewee] ('DELETE FROM "teammember" WHERE ("teammember"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,059 [75] [DEBUG] [peewee] ('DELETE FROM "star" WHERE ("star"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,059 [75] [DEBUG] [peewee] ('DELETE FROM "repositorypermission" WHERE ("repositorypermission"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,060 [75] [DEBUG] [peewee] ('DELETE FROM "repositoryautoprunepolicy" WHERE ("repositoryautoprunepolicy"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,060 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."delegate_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,061 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."activating_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,061 [75] [DEBUG] [peewee] ('DELETE FROM "permissionprototype" WHERE ("permissionprototype"."org_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,062 [75] [DEBUG] [peewee] ('DELETE FROM "oauthassignedtoken" WHERE ("oauthassignedtoken"."assigned_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,062 [75] [DEBUG] [peewee] ('DELETE FROM "oauthaccesstoken" WHERE ("oauthaccesstoken"."authorized_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,063 [75] [DEBUG] [peewee] ('DELETE FROM "userregion" WHERE ("userregion"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,063 [75] [DEBUG] [peewee] ('DELETE FROM "userprompt" WHERE ("userprompt"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,064 [75] [DEBUG] [peewee] ('DELETE FROM "userorganizationquota" WHERE ("userorganizationquota"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,064 [75] [DEBUG] [peewee] ('DELETE FROM "team" WHERE ("team"."organization_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,065 [75] [DEBUG] [peewee] ('DELETE FROM "servicekeyapproval" WHERE ("servicekeyapproval"."approver_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,065 [75] [DEBUG] [peewee] ('DELETE FROM "robotaccounttoken" WHERE ("robotaccounttoken"."robot_account_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,065 [75] [DEBUG] [peewee] ('DELETE FROM "robotaccountmetadata" WHERE ("robotaccountmetadata"."robot_account_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,066 [75] [DEBUG] [peewee] ('DELETE FROM "repository" WHERE ("repository"."namespace_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,067 [75] [DEBUG] [peewee] ('DELETE FROM "redhatsubscriptions" WHERE ("redhatsubscriptions"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,067 [75] [DEBUG] [peewee] ('DELETE FROM "quotanamespacesize" WHERE ("quotanamespacesize"."namespace_user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,067 [75] [DEBUG] [peewee] ('DELETE FROM "proxycacheconfig" WHERE ("proxycacheconfig"."organization_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,068 [75] [DEBUG] [peewee] ('DELETE FROM "organizationrhskus" WHERE ("organizationrhskus"."org_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,068 [75] [DEBUG] [peewee] ('DELETE FROM "organizationrhskus" WHERE ("organizationrhskus"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,070 [75] [DEBUG] [peewee] ('DELETE FROM "oauthapplication" WHERE ("oauthapplication"."organization_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,070 [75] [DEBUG] [peewee] ('DELETE FROM "notification" WHERE ("notification"."target_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,071 [75] [DEBUG] [peewee] ('DELETE FROM "namespacegeorestriction" WHERE ("namespacegeorestriction"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,071 [75] [DEBUG] [peewee] ('DELETE FROM "namespaceautoprunepolicy" WHERE ("namespaceautoprunepolicy"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,071 [75] [DEBUG] [peewee] ('DELETE FROM "federatedlogin" WHERE ("federatedlogin"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,072 [75] [DEBUG] [peewee] ('DELETE FROM "emailconfirmation" WHERE ("emailconfirmation"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,072 [75] [DEBUG] [peewee] ('DELETE FROM "deletednamespace" WHERE ("deletednamespace"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,074 [75] [DEBUG] [peewee] ('DELETE FROM "autoprunetaskstatus" WHERE ("autoprunetaskstatus"."namespace_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,074 [75] [DEBUG] [peewee] ('DELETE FROM "appspecificauthtoken" WHERE ("appspecificauthtoken"."user_id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,075 [75] [DEBUG] [peewee] ('DELETE FROM "user" WHERE ("user"."id" = %s)', [5]) namespacegcworker stdout | 2025-11-04 09:06:57,082 [75] [DEBUG] [util.locking] Releasing lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:57,082 [75] [DEBUG] [redis_lock.release] Releasing 'lock:LARGE_GARBAGE_COLLECTION'. namespacegcworker stdout | 2025-11-04 09:06:57,083 [75] [DEBUG] [util.locking] Released lock LARGE_GARBAGE_COLLECTION namespacegcworker stdout | 2025-11-04 09:06:57,083 [75] [DEBUG] [peewee] ('DELETE FROM "queueitem" WHERE ("queueitem"."id" = %s)', [1]) namespacegcworker stdout | 2025-11-04 09:06:57,085 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 57, 85277), True, datetime.datetime(2025, 11, 4, 9, 6, 57, 85277), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:06:57,087 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:06:57,087 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:06:57,087 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:07:53 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:06:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:06:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:07.766253+00:00 (in 9.999567 seconds) notificationworker stdout | 2025-11-04 09:06:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:07 GMT)" (scheduled at 2025-11-04 09:06:57.766253+00:00) notificationworker stdout | 2025-11-04 09:06:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:06:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 57, 767020), True, datetime.datetime(2025, 11, 4, 9, 6, 57, 767020), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:06:57,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:06:57,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:06:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:06:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:06:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:09.676299+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 09:06:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:09 GMT)" (scheduled at 2025-11-04 09:06:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:06:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:06:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 6, 59, 677059), True, datetime.datetime(2025, 11, 4, 9, 6, 59, 677059), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:06:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:06:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:06:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:06:59,748 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:06:59,848 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:52.159714+00:00 (in 52.072857 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:00 GMT)" (scheduled at 2025-11-04 09:07:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,093 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:07:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:00 GMT)" executed successfully builder stdout | 2025-11-04 09:07:00,139 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:07:00,949 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:07:01,143 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:19.706478+00:00 (in 17.998722 seconds) securityworker stdout | 2025-11-04 09:07:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:31 GMT)" (scheduled at 2025-11-04 09:07:01.707296+00:00) securityworker stdout | 2025-11-04 09:07:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:01,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,716 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,716 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:01,716 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:01,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:01,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:01,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:01,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:01,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 1, 711471), 1, 49]) securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 1, 711471), 1, 49]) securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:01,723 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:01,723 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:07:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:07:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:32.081579+00:00 (in 29.999567 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:07:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:32 GMT)" (scheduled at 2025-11-04 09:07:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:07:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:07:02,084 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:07:02,084 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:07:02,347 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:07:03,946 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:07:04,048 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:05,135 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:07:05,629 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:07:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:07:06,267 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:40.180734+00:00 (in 33.913574 seconds) repositorygcworker stdout | 2025-11-04 09:07:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:06 GMT)" (scheduled at 2025-11-04 09:07:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:07:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:07:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:07:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:28.569177+00:00 (in 21.999211 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:06 GMT)" (scheduled at 2025-11-04 09:07:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:07:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:07:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:07:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:37.077358+00:00 (in 29.999518 seconds) autopruneworker stdout | 2025-11-04 09:07:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:37 GMT)" (scheduled at 2025-11-04 09:07:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:07:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243627082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:07:07,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:07:07,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:07:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:07:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:25.661677+00:00 (in 17.997744 seconds) gcworker stdout | 2025-11-04 09:07:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:37 GMT)" (scheduled at 2025-11-04 09:07:07.663502+00:00) gcworker stdout | 2025-11-04 09:07:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:07:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246927670, None, 1, 0]) gcworker stdout | 2025-11-04 09:07:07,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:07,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:17.766253+00:00 (in 9.999566 seconds) notificationworker stdout | 2025-11-04 09:07:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:17 GMT)" (scheduled at 2025-11-04 09:07:07.766253+00:00) notificationworker stdout | 2025-11-04 09:07:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 7, 767055), True, datetime.datetime(2025, 11, 4, 9, 7, 7, 767055), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:07:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:07:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:07.770322+00:00 (in 59.999594 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:07:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:07 GMT)" (scheduled at 2025-11-04 09:07:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:07:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:07:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:07:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:07:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:07:08,145 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:07:09,251 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:09,529 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:19.676299+00:00 (in 9.999589 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:19 GMT)" (scheduled at 2025-11-04 09:07:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 9, 677026), True, datetime.datetime(2025, 11, 4, 9, 7, 9, 677026), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:09,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:09,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:09,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:19 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:07:09,942 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:09,946 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:10,065 [257] [DEBUG] [app] Starting request: urn:request:74cc6563-9232-485c-adb1-3cf26f81bf01 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:10,066 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:10,068 [250] [DEBUG] [app] Starting request: urn:request:84f11685-4746-4c43-8839-d7ed851cdc28 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:10,069 [250] [DEBUG] [app] Ending request: urn:request:84f11685-4746-4c43-8839-d7ed851cdc28 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:84f11685-4746-4c43-8839-d7ed851cdc28', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:10,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:07:10,070 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:10,070 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:10,072 [258] [DEBUG] [app] Starting request: urn:request:20304f33-017f-4331-8726-a5c3d1f7cc58 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:10,072 [258] [DEBUG] [app] Ending request: urn:request:20304f33-017f-4331-8726-a5c3d1f7cc58 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:20304f33-017f-4331-8726-a5c3d1f7cc58', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:10,072 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:10,073 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:10,073 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:10,073 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:10,073 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:10,078 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:10,078 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:10,083 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:10,084 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:10,084 [257] [DEBUG] [app] Ending request: urn:request:74cc6563-9232-485c-adb1-3cf26f81bf01 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:74cc6563-9232-485c-adb1-3cf26f81bf01', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:10,084 [257] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:07:10,085 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:07:10,122 [261] [DEBUG] [app] Starting request: urn:request:0c58a2e1-37b4-4e71-907b-883070fa1233 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:10,123 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:10,127 [259] [DEBUG] [app] Starting request: urn:request:15cc8bdf-cc99-44c3-8eb7-c909d4370b8d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:10,128 [259] [DEBUG] [app] Ending request: urn:request:15cc8bdf-cc99-44c3-8eb7-c909d4370b8d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:15cc8bdf-cc99-44c3-8eb7-c909d4370b8d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.005 162 0.005) gunicorn-registry stdout | 2025-11-04 09:07:10,129 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:10,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:10,131 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:10,133 [254] [DEBUG] [app] Starting request: urn:request:cb5b9327-f1b5-4077-bcbc-f5f426f1d961 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:10,133 [254] [DEBUG] [app] Ending request: urn:request:cb5b9327-f1b5-4077-bcbc-f5f426f1d961 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cb5b9327-f1b5-4077-bcbc-f5f426f1d961', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:10,134 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:10,134 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:10,134 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:10,134 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:10,134 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:10,138 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:10,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:10,142 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:10,143 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:10,144 [261] [DEBUG] [app] Ending request: urn:request:0c58a2e1-37b4-4e71-907b-883070fa1233 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0c58a2e1-37b4-4e71-907b-883070fa1233', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:10,144 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:10,144 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) quotaregistrysizeworker stdout | 2025-11-04 09:07:10,748 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:07:11,042 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:07:11,340 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:11,955 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:11,959 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:11,959 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:07:12,449 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:07:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:38.480592+00:00 (in 25.998063 seconds) exportactionlogsworker stdout | 2025-11-04 09:07:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:12 GMT)" (scheduled at 2025-11-04 09:07:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:07:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:07:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:12 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:07:12,846 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:13,239 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:14,042 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:14,464 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:14,464 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:14,466 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:14,523 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:14,524 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:07:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:07:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:44.980431+00:00 (in 29.999531 seconds) buildlogsarchiver stdout | 2025-11-04 09:07:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:44 GMT)" (scheduled at 2025-11-04 09:07:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:07:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 7, 14, 981208), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:07:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:07:14,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:07:15,950 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,034 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,048 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,051 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,055 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,056 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,057 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,058 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:16,059 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:07:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:17,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:19.767846+00:00 (in 2.000686 seconds) notificationworker stdout | 2025-11-04 09:07:17,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:27 GMT)" (scheduled at 2025-11-04 09:07:17.766253+00:00) notificationworker stdout | 2025-11-04 09:07:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 17, 767564), True, datetime.datetime(2025, 11, 4, 9, 7, 17, 767564), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:17,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:07:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:29.676299+00:00 (in 9.999607 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:29 GMT)" (scheduled at 2025-11-04 09:07:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 19, 677019), True, datetime.datetime(2025, 11, 4, 9, 7, 19, 677019), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:07:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:31.707296+00:00 (in 12.000415 seconds) securityworker stdout | 2025-11-04 09:07:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:49 GMT)" (scheduled at 2025-11-04 09:07:19.706478+00:00) securityworker stdout | 2025-11-04 09:07:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:19,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:49 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:19,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:19,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:27.766253+00:00 (in 7.997949 seconds) notificationworker stdout | 2025-11-04 09:07:19,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:19 GMT)" (scheduled at 2025-11-04 09:07:19.767846+00:00) notificationworker stdout | 2025-11-04 09:07:19,769 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), 'notification/%']) notificationworker stdout | 2025-11-04 09:07:19,776 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), True, datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:07:19,777 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), True, datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 7, 19, 768658), 'notification/%']) notificationworker stdout | 2025-11-04 09:07:19,778 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:19,779 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:07:20,853 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:25,065 [257] [DEBUG] [app] Starting request: urn:request:9b4ef0fa-14f7-46ca-9445-115e54cd55df (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:25,066 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:25,068 [260] [DEBUG] [app] Starting request: urn:request:8c04476f-1801-4735-818a-bb905fc76bd6 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:25,069 [260] [DEBUG] [app] Ending request: urn:request:8c04476f-1801-4735-818a-bb905fc76bd6 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8c04476f-1801-4735-818a-bb905fc76bd6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:07:25,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:25,069 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:25,078 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:25,080 [261] [DEBUG] [app] Starting request: urn:request:0418d924-5255-44e0-a4b9-82da9e6ec4ef (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:25,080 [261] [DEBUG] [app] Ending request: urn:request:0418d924-5255-44e0-a4b9-82da9e6ec4ef (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0418d924-5255-44e0-a4b9-82da9e6ec4ef', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:25,080 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:07:25,080 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:25,081 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:25,081 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:25,081 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:25,085 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:25,085 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:25,089 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:25,090 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:25,091 [257] [DEBUG] [app] Ending request: urn:request:9b4ef0fa-14f7-46ca-9445-115e54cd55df (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9b4ef0fa-14f7-46ca-9445-115e54cd55df', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:25,091 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:25,091 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.027) gunicorn-web stdout | 2025-11-04 09:07:25,123 [254] [DEBUG] [app] Starting request: urn:request:c2041c20-5400-432b-ad36-152a5af4c574 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:25,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:25,126 [260] [DEBUG] [app] Starting request: urn:request:64a7f7c9-502c-437b-b8cc-134b99ea4e24 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:25,126 [260] [DEBUG] [app] Ending request: urn:request:64a7f7c9-502c-437b-b8cc-134b99ea4e24 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:64a7f7c9-502c-437b-b8cc-134b99ea4e24', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:07:25,126 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:25,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:25,128 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:25,129 [254] [DEBUG] [app] Starting request: urn:request:1348d1d5-ce6c-4c90-b16a-88fb764a497f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:25,129 [254] [DEBUG] [app] Ending request: urn:request:1348d1d5-ce6c-4c90-b16a-88fb764a497f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1348d1d5-ce6c-4c90-b16a-88fb764a497f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:25,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:25,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:25,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:25,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:25,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:25,134 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:25,134 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:25,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:25,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:25,140 [254] [DEBUG] [app] Ending request: urn:request:c2041c20-5400-432b-ad36-152a5af4c574 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c2041c20-5400-432b-ad36-152a5af4c574', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:25,140 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:25,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) gunicorn-registry stdout | 2025-11-04 09:07:25,491 [255] [DEBUG] [app] Starting request: urn:request:28a231b4-45e8-41ac-b76c-c21b8f37d1d6 (/v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a) {'X-Forwarded-For': '183.241.154.170, 10.129.4.13'} gunicorn-registry stdout | 2025-11-04 09:07:25,491 [255] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'manifest_ref': 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'namespace_name': 'superorg', 'repo_name': 'repo1'} gunicorn-registry stdout | 2025-11-04 09:07:25,491 [255] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MjQ0LCJpYXQiOjE3NjIyNDcyNDQsImV4cCI6MTc2MjI1MDg0NCwic3ViIjoiKGFub255bW91cykiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InN1cGVyb3JnL3JlcG8xIiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InN1cGVyb3JnL3JlcG8xIjoiJGRpc2FibGVkIn0sImNvbS5hcG9zdGlsbGUucm9vdCI6IiRkaXNhYmxlZCJ9fQ.EGAY-u_f4omkYzZLYcAtywi7N5Jqhp3wqwWGPUucFUnE4Q_Ca2Gj-asvAkHjfneRB_AF-gHm7OfcK1wkWaUmsM5dtaKoxdismDFBKzsH8Rp_0-Ma7llSsImf4HCNgl8RIRXPbV5sd2TVXGoGSzIiSw7ipSBT9kCSqpxflxoBceWsZYaxnJa85w3MsxeJColgnCzJadp-So5-NPgo_xC41IKTz36_QeEIGhP4EiTde6x2MiDwRbFkqkjRWp18hiW1VGuTMr_H5OG7iafZatzevZQLWyDH53OBEvPa1AEzAUiGEq_GUe6-dF2Bj-hp4B0UIYheHRM_o-0TGEnnreNsJw gunicorn-registry stdout | 2025-11-04 09:07:25,492 [255] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MjQ0LCJpYXQiOjE3NjIyNDcyNDQsImV4cCI6MTc2MjI1MDg0NCwic3ViIjoiKGFub255bW91cykiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InN1cGVyb3JnL3JlcG8xIiwiYWN0aW9ucyI6WyJwdWxsIl19XSwiY29udGV4dCI6eyJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InN1cGVyb3JnL3JlcG8xIjoiJGRpc2FibGVkIn0sImNvbS5hcG9zdGlsbGUucm9vdCI6IiRkaXNhYmxlZCJ9fQ.EGAY-u_f4omkYzZLYcAtywi7N5Jqhp3wqwWGPUucFUnE4Q_Ca2Gj-asvAkHjfneRB_AF-gHm7OfcK1wkWaUmsM5dtaKoxdismDFBKzsH8Rp_0-Ma7llSsImf4HCNgl8RIRXPbV5sd2TVXGoGSzIiSw7ipSBT9kCSqpxflxoBceWsZYaxnJa85w3MsxeJColgnCzJadp-So5-NPgo_xC41IKTz36_QeEIGhP4EiTde6x2MiDwRbFkqkjRWp18hiW1VGuTMr_H5OG7iafZatzevZQLWyDH53OBEvPa1AEzAUiGEq_GUe6-dF2Bj-hp4B0UIYheHRM_o-0TGEnnreNsJw gunicorn-registry stdout | 2025-11-04 09:07:25,494 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 7, 25, 493099), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 7, 25, 493131), 'quay', None, datetime.datetime(2025, 11, 3, 9, 7, 25, 493154), datetime.datetime(2025, 10, 28, 9, 7, 25, 493171), None]) gunicorn-registry stdout | 2025-11-04 09:07:25,494 [255] [DEBUG] [peewee.pool] No connection available in pool. gunicorn-registry stdout | 2025-11-04 09:07:25,498 [255] [DEBUG] [peewee.pool] Created new connection 140331977079232. gunicorn-registry stdout | 2025-11-04 09:07:25,505 [255] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:07:25,505 [255] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: (anonymous) gunicorn-registry stdout | 2025-11-04 09:07:25,505 [255] [DEBUG] [auth.registry_jwt_auth] Identity changed to (anonymous) gunicorn-registry stdout | 2025-11-04 09:07:25,505 [255] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'com.apostille.roots': {'superorg/repo1': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:07:25,505 [255] [DEBUG] [endpoints.v2] Checking permission for repo: superorg/repo1 gunicorn-registry stdout | 2025-11-04 09:07:25,506 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['superorg', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,508 [255] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['superorg', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:07:25,508 [255] [DEBUG] [data.cache.cache_key] Loading repository lookup from cache_key: repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a gunicorn-registry stdout | 2025-11-04 09:07:25,508 [255] [DEBUG] [data.cache.impl] Checking cache for key repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a gunicorn-registry stdout | 2025-11-04 09:07:25,509 [255] [DEBUG] [data.cache.impl] Found no result in cache for key repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a; calling loader gunicorn-registry stdout | 2025-11-04 09:07:25,516 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['superorg', 'repo1', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,517 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,518 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,519 [255] [DEBUG] [data.cache.impl] Got loaded result for key repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a: {'id': 9, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}} gunicorn-registry stdout | 2025-11-04 09:07:25,519 [255] [DEBUG] [data.cache.impl] Caching loaded result for key repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a with expiration {'id': 9, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:07:25,519 [255] [DEBUG] [data.cache.impl] Cached loaded result for key repository_lookup_superorg_repo1_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a with expiration {'id': 9, 'visibility': {'id': 1, 'name': 'public'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:07:25,520 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:07:25,521 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['public', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,521 [255] [DEBUG] [data.cache.impl] Checking cache for key repository_manifest__9_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a gunicorn-registry stdout | 2025-11-04 09:07:25,522 [255] [DEBUG] [data.cache.impl] Found no result in cache for key repository_manifest__9_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a; calling loader gunicorn-registry stdout | 2025-11-04 09:07:25,523 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "tag" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t1"."repository_id" = %s) AND ("t1"."digest" = %s)) AND (("t2"."lifetime_end_ms" IS %s) OR ("t2"."lifetime_end_ms" > %s))) LIMIT %s OFFSET %s', [9, 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', None, 1762247245522, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,525 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "mediatype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:07:25,526 [255] [DEBUG] [data.cache.impl] Got loaded result for key repository_manifest__9_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a: {'digest': 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1472,\n "digest": "sha256:3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 828678,\n "digest": "sha256:87379020f3b6731a4b64976e614d305f5c121d153c049d14ba600ff24bbac012"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 828678, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 35, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 9, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}} gunicorn-registry stdout | 2025-11-04 09:07:25,526 [255] [DEBUG] [data.cache.impl] Caching loaded result for key repository_manifest__9_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a with expiration {'digest': 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1472,\n "digest": "sha256:3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 828678,\n "digest": "sha256:87379020f3b6731a4b64976e614d305f5c121d153c049d14ba600ff24bbac012"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 828678, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 35, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 9, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}}: 300s gunicorn-registry stdout | 2025-11-04 09:07:25,527 [255] [DEBUG] [data.cache.impl] Cached loaded result for key repository_manifest__9_sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a with expiration {'digest': 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1472,\n "digest": "sha256:3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 828678,\n "digest": "sha256:87379020f3b6731a4b64976e614d305f5c121d153c049d14ba600ff24bbac012"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 828678, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 35, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 9, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}}: 300s gunicorn-registry stdout | 2025-11-04 09:07:25,527 [255] [DEBUG] [util.audit] Checking publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:07:25,527 [255] [DEBUG] [util.audit] Logging the pull_repo to analytics engine gunicorn-registry stdout | 2025-11-04 09:07:25,527 [255] [DEBUG] [util.audit] Resolving IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:07:25,529 [255] [DEBUG] [util.audit] Resolved IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:07:25,529 [255] [DEBUG] [util.audit] Logging the pull_repo to logs system gunicorn-registry stdout | 2025-11-04 09:07:25,530 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['superorg', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:07:25,530 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "logentrykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:07:25,532 [255] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [41, 4, None, 9, datetime.datetime(2025, 11, 4, 9, 7, 25, 529652), '10.129.4.13', '{"repo": "repo1", "namespace": "superorg", "user-agent": "containers/5.34.3 (github.com/containers/image)", "manifest_digest": "sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a", "resolved_ip": {"provider": "internet", "service": null, "sync_token": "1645662201", "country_iso_code": null, "aws_region": null, "continent": null}}']) gunicorn-registry stdout | 2025-11-04 09:07:25,534 [255] [DEBUG] [util.audit] Track and log of pull_repo complete gunicorn-registry stdout | 2025-11-04 09:07:25,536 [255] [DEBUG] [app] Ending request: urn:request:28a231b4-45e8-41ac-b76c-c21b8f37d1d6 (/v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a) {'endpoint': 'v2.fetch_manifest_by_digest', 'request_id': 'urn:request:28a231b4-45e8-41ac-b76c-c21b8f37d1d6', 'remote_addr': '10.129.4.13', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'path': '/v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:07:25,537 [255] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:07:25,537 [255] [DEBUG] [peewee.pool] Returning 140331977079232 to pool. nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:07:25 +0000] "GET /v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a HTTP/1.1" 200 527 "-" "containers/5.34.3 (github.com/containers/image)" (0.050 1977 0.050) gunicorn-registry stdout | 2025-11-04 09:07:25,537 [255] [INFO] [gunicorn.access] 10.129.4.13 - - [04/Nov/2025:09:07:25 +0000] "GET /v2/superorg/repo1/manifests/sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a HTTP/1.1" 200 527 "-" "containers/5.34.3 (github.com/containers/image)" gunicorn-registry stdout | 2025-11-04 09:07:25,545 [255] [DEBUG] [util.pullmetrics] Tracked manifest pull: repo_id=9 digest=sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a notificationworker stdout | 2025-11-04 09:07:25,565 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:37.663502+00:00 (in 12.001420 seconds) gcworker stdout | 2025-11-04 09:07:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:55 GMT)" (scheduled at 2025-11-04 09:07:25.661677+00:00) gcworker stdout | 2025-11-04 09:07:25,662 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037645662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:07:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:07:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:07:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:25,952 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:26,820 [258] [DEBUG] [app] Starting request: urn:request:3886dc84-b77c-4454-9e26-40eb72086e98 (/_storage_proxy_auth) {'X-Forwarded-For': '183.241.154.170'} gunicorn-web stdout | 2025-11-04 09:07:26,821 [258] [DEBUG] [storage.downloadproxy] Got token b'eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MjQ1LCJpYXQiOjE3NjIyNDcyNDUsImV4cCI6MTc2MjI0Nzg0NSwic3ViIjoic3RvcmFnZXByb3h5IiwiYWNjZXNzIjpbeyJ0eXBlIjoic3RvcmFnZXByb3h5IiwidXJpIjoicXVheS1kYXRhc3RvcmUtOWVjOGM2NGYtNjM0ZC00NmNmLWEyOGQtOWM0ZjAyMDA2YTg0L2RhdGFzdG9yYWdlL3JlZ2lzdHJ5L3NoYTI1Ni8zYy8zYzE5YmFmZWQyMjM1NWUxMWE2MDhjNGI2MTNkODdkMDZiOWNkZDM3ZDM3OGU2ZTAxNzZjYmM4ZTcxNDRkNWM2P0FXU0FjY2Vzc0tleUlkPWU0YnMxVlJVNXV6VmdiVjB2bzBtJlNpZ25hdHVyZT11Y1FsYlFubXFGYkR2JTJGUFRUWVp2aHc2QjJNayUzRCZFeHBpcmVzPTE3NjIyNDc4NDUiLCJob3N0IjoiczMub3BlbnNoaWZ0LXN0b3JhZ2Uuc3ZjLmNsdXN0ZXIubG9jYWw6NDQzIiwic2NoZW1lIjoiaHR0cHMifV0sImNvbnRleHQiOnt9fQ.Ym4089IwXY0jV6SMZytGBGhnIGGVkr2F-sB3-7Nnh37nXTuh2xmtxo8Q5NKAmp-KNHifn2aDYKEoeeyg17QJjRERdlYeQM86YgAo90G5RbFF5C4TQR03T915bUdh1SAxnGySB01Jh_bi7LvjfjHXFVNFrYJf5BwJBYg79bv_SvnIaJEnJ7U7k_Dl93AARmpTdSOjPJmYpa1JXcN67lfNy2jqxpgtKmI7XBtttSjsbU-WS62Y9nGVu-h5O_nF7xdSixL9R1X1EaSlFIPS88BAMOM02MOLYdSRlnwRO0Z3mL6Hv1Xnv3kb0K_DYQ5Fcw3Sx1gJ0XkD3frjTKmeisVwfw' for storage proxy auth request /_storage_proxy/ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklsbHBSVFpQYTNSRll6Sm9kRWcyWld4UU4xbE5TVkZKTm1kcU9YQklielJUUjA1TmN6YzBUelJuTFRBaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM01qUTFMQ0pwWVhRaU9qRTNOakl5TkRjeU5EVXNJbVY0Y0NJNk1UYzJNakkwTnpnME5Td2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk4ell5OHpZekU1WW1GbVpXUXlNak0xTldVeE1XRTJNRGhqTkdJMk1UTmtPRGRrTURaaU9XTmtaRE0zWkRNM09HVTJaVEF4TnpaalltTTRaVGN4TkRSa05XTTJQMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDExWTFGc1lsRnViWEZHWWtSMkpUSkdVRlJVV1ZwMmFIYzJRakpOYXlVelJDWkZlSEJwY21WelBURTNOakl5TkRjNE5EVWlMQ0pvYjNOMElqb2ljek11YjNCbGJuTm9hV1owTFhOMGIzSmhaMlV1YzNaakxtTnNkWE4wWlhJdWJHOWpZV3c2TkRReklpd2ljMk5vWlcxbElqb2lhSFIwY0hNaWZWMHNJbU52Ym5SbGVIUWlPbnQ5ZlEuWW00MDg5SXdYWTBqVjZTTVp5dEdCR2huSUdHVmtyMkYtc0IzLTdObmgzN25YVHVoMnhtdHhvOFE1TktBbXAtS05IaWZuMmFEWUtFb2VleWcxN1FKalJFUmRsWWVRTTg2WWdBbzkwRzVSYkZGNUM0VFFSMDNUOTE1YlVkaDFTQXhuR3lTQjAxSmhfYmk3THZqZmpIWEZWTkZyWUpmNUJ3SkJZZzc5YnZfU3ZuSWFKRW5KN1U3a19EbDkzQUFSbXBUZFNPalBKbVlwYTFKWGNONjdsZk55MmpxeHBndEttSTdYQnR0dFNqc2JVLVdTNjJZOW5HVnUtaDVPX25GN3hkU2l4TDlSMVgxRWFTbEZJUFM4OEJBTU9NMDJNT0xZZFNSbG53Uk8wWjNtTDZIdjFYbnYza2IwS19EWVE1RmN3M1N4MWdKMFhrRDNmcmpUS21laXNWd2Z3/https/s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/3c/3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=ucQlbQnmqFbDv%2FPTTYZvhw6B2Mk%3D&Expires=1762247845 with parts ['ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklsbHBSVFpQYTNSRll6Sm9kRWcyWld4UU4xbE5TVkZKTm1kcU9YQklielJUUjA1TmN6YzBUelJuTFRBaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM01qUTFMQ0pwWVhRaU9qRTNOakl5TkRjeU5EVXNJbVY0Y0NJNk1UYzJNakkwTnpnME5Td2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk4ell5OHpZekU1WW1GbVpXUXlNak0xTldVeE1XRTJNRGhqTkdJMk1UTmtPRGRrTURaaU9XTmtaRE0zWkRNM09HVTJaVEF4TnpaalltTTRaVGN4TkRSa05XTTJQMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDExWTFGc1lsRnViWEZHWWtSMkpUSkdVRlJVV1ZwMmFIYzJRakpOYXlVelJDWkZlSEJwY21WelBURTNOakl5TkRjNE5EVWlMQ0pvYjNOMElqb2ljek11YjNCbGJuTm9hV1owTFhOMGIzSmhaMlV1YzNaakxtTnNkWE4wWlhJdWJHOWpZV3c2TkRReklpd2ljMk5vWlcxbElqb2lhSFIwY0hNaWZWMHNJbU52Ym5SbGVIUWlPbnQ5ZlEuWW00MDg5SXdYWTBqVjZTTVp5dEdCR2huSUdHVmtyMkYtc0IzLTdObmgzN25YVHVoMnhtdHhvOFE1TktBbXAtS05IaWZuMmFEWUtFb2VleWcxN1FKalJFUmRsWWVRTTg2WWdBbzkwRzVSYkZGNUM0VFFSMDNUOTE1YlVkaDFTQXhuR3lTQjAxSmhfYmk3THZqZmpIWEZWTkZyWUpmNUJ3SkJZZzc5YnZfU3ZuSWFKRW5KN1U3a19EbDkzQUFSbXBUZFNPalBKbVlwYTFKWGNONjdsZk55MmpxeHBndEttSTdYQnR0dFNqc2JVLVdTNjJZOW5HVnUtaDVPX25GN3hkU2l4TDlSMVgxRWFTbEZJUFM4OEJBTU9NMDJNT0xZZFNSbG53Uk8wWjNtTDZIdjFYbnYza2IwS19EWVE1RmN3M1N4MWdKMFhrRDNmcmpUS21laXNWd2Z3', 'https', 's3.openshift-storage.svc.cluster.local:443', 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/3c/3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=ucQlbQnmqFbDv%2FPTTYZvhw6B2Mk%3D&Expires=1762247845'] gunicorn-web stdout | 2025-11-04 09:07:26,823 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 7, 26, 821868), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 7, 26, 822381), 'quay', None, datetime.datetime(2025, 11, 3, 9, 7, 26, 822428), datetime.datetime(2025, 10, 28, 9, 7, 26, 822451), None]) gunicorn-web stdout | 2025-11-04 09:07:26,834 [258] [DEBUG] [app] Ending request: urn:request:3886dc84-b77c-4454-9e26-40eb72086e98 (/_storage_proxy_auth) {'endpoint': '_storage_proxy_auth', 'request_id': 'urn:request:3886dc84-b77c-4454-9e26-40eb72086e98', 'remote_addr': '183.241.154.170', 'http_method': 'GET', 'original_url': 'https://web_app_server/_storage_proxy_auth', 'path': '/_storage_proxy_auth', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-web stdout | 2025-11-04 09:07:26,834 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:26,834 [258] [INFO] [gunicorn.access] 183.241.154.170 - - [04/Nov/2025:09:07:26 +0000] "GET /_storage_proxy_auth HTTP/1.0" 200 2 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/superorg/repo1/blobs/sha256:3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6" "containers/5.34.3 (github.com/containers/image)" dnsmasq stderr | dnsmasq: nameserver 172.30.0.10 refused to do a recursive query nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:07:26 +0000] "GET /_storage_proxy/ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklsbHBSVFpQYTNSRll6Sm9kRWcyWld4UU4xbE5TVkZKTm1kcU9YQklielJUUjA1TmN6YzBUelJuTFRBaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM01qUTFMQ0pwWVhRaU9qRTNOakl5TkRjeU5EVXNJbVY0Y0NJNk1UYzJNakkwTnpnME5Td2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk4ell5OHpZekU1WW1GbVpXUXlNak0xTldVeE1XRTJNRGhqTkdJMk1UTmtPRGRrTURaaU9XTmtaRE0zWkRNM09HVTJaVEF4TnpaalltTTRaVGN4TkRSa05XTTJQMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDExWTFGc1lsRnViWEZHWWtSMkpUSkdVRlJVV1ZwMmFIYzJRakpOYXlVelJDWkZlSEJwY21WelBURTNOakl5TkRjNE5EVWlMQ0pvYjNOMElqb2ljek11YjNCbGJuTm9hV1owTFhOMGIzSmhaMlV1YzNaakxtTnNkWE4wWlhJdWJHOWpZV3c2TkRReklpd2ljMk5vWlcxbElqb2lhSFIwY0hNaWZWMHNJbU52Ym5SbGVIUWlPbnQ5ZlEuWW00MDg5SXdYWTBqVjZTTVp5dEdCR2huSUdHVmtyMkYtc0IzLTdObmgzN25YVHVoMnhtdHhvOFE1TktBbXAtS05IaWZuMmFEWUtFb2VleWcxN1FKalJFUmRsWWVRTTg2WWdBbzkwRzVSYkZGNUM0VFFSMDNUOTE1YlVkaDFTQXhuR3lTQjAxSmhfYmk3THZqZmpIWEZWTkZyWUpmNUJ3SkJZZzc5YnZfU3ZuSWFKRW5KN1U3a19EbDkzQUFSbXBUZFNPalBKbVlwYTFKWGNONjdsZk55MmpxeHBndEttSTdYQnR0dFNqc2JVLVdTNjJZOW5HVnUtaDVPX25GN3hkU2l4TDlSMVgxRWFTbEZJUFM4OEJBTU9NMDJNT0xZZFNSbG53Uk8wWjNtTDZIdjFYbnYza2IwS19EWVE1RmN3M1N4MWdKMFhrRDNmcmpUS21laXNWd2Z3/https/s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/3c/3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=ucQlbQnmqFbDv%2FPTTYZvhw6B2Mk%3D&Expires=1762247845 HTTP/1.1" 200 674 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/superorg/repo1/blobs/sha256:3c19bafed22355e11a608c4b613d87d06b9cdd37d378e6e0176cbc8e7144d5c6" "containers/5.34.3 (github.com/containers/image)" (0.050 3615 0.035) notificationworker stdout | 2025-11-04 09:07:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:27,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:27.768179+00:00 (in 0.001010 seconds) notificationworker stdout | 2025-11-04 09:07:27,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:37 GMT)" (scheduled at 2025-11-04 09:07:27.766253+00:00) notificationworker stdout | 2025-11-04 09:07:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 27, 767535), True, datetime.datetime(2025, 11, 4, 9, 7, 27, 767535), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:37.766253+00:00 (in 9.997752 seconds) notificationworker stdout | 2025-11-04 09:07:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:27 GMT)" (scheduled at 2025-11-04 09:07:27.768179+00:00) notificationworker stdout | 2025-11-04 09:07:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:07:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:28,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:28,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:33.568037+00:00 (in 4.998365 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:28,569 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:28 GMT)" (scheduled at 2025-11-04 09:07:28.569177+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:28,570 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:07:28,577 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), True, datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:07:28,578 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), True, datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 7, 28, 570025), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:07:28,579 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:07:28,579 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:28 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:07:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:39.676299+00:00 (in 9.999611 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:39 GMT)" (scheduled at 2025-11-04 09:07:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 29, 677041), True, datetime.datetime(2025, 11, 4, 9, 7, 29, 677041), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:29,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:07:29,762 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:07:29,863 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:07:30,157 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:07:30,963 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:07:31,157 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:49.706478+00:00 (in 17.998793 seconds) securityworker stdout | 2025-11-04 09:07:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:01 GMT)" (scheduled at 2025-11-04 09:07:31.707296+00:00) securityworker stdout | 2025-11-04 09:07:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:31,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:31,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:31,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:31,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 31, 711733), 1, 49]) securityworker stdout | 2025-11-04 09:07:31,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:31,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 31, 711733), 1, 49]) securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:31,725 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:07:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:07:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:07:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:02.081579+00:00 (in 29.999586 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:02 GMT)" (scheduled at 2025-11-04 09:07:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:07:32,083 [79] [INFO] [__main__] RedisFlushWorker: Processing 2 Redis keys pullstatsredisflushworker stdout | 2025-11-04 09:07:32,092 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."tag_name", "t1"."tag_pull_count", "t1"."last_tag_pull_date", "t1"."current_manifest_digest" FROM "tagpullstatistics" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."tag_name" = %s))', [9, 'busybox']) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,094 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,095 [79] [DEBUG] [peewee] ('UPDATE "tagpullstatistics" SET "repository_id" = %s, "tag_name" = %s, "tag_pull_count" = %s, "last_tag_pull_date" = %s, "current_manifest_digest" = %s WHERE ("tagpullstatistics"."id" = %s)', [9, 'busybox', 1, datetime.datetime(2025, 11, 4, 9, 7, 25), 'sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83', 2]) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,097 [79] [INFO] [__main__] RedisFlushWorker: Successfully updated 1/1 tag statistics pullstatsredisflushworker stdout | 2025-11-04 09:07:32,098 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_digest", "t1"."manifest_pull_count", "t1"."last_manifest_pull_date" FROM "manifestpullstatistics" AS "t1" WHERE ((("t1"."repository_id" = %s) AND ("t1"."manifest_digest" = %s)) OR (("t1"."repository_id" = %s) AND ("t1"."manifest_digest" = %s)))', [9, 'sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83', 9, 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a']) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,099 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [9, 1, 0]) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,099 [79] [DEBUG] [peewee] ('UPDATE "manifestpullstatistics" SET "repository_id" = %s, "manifest_digest" = %s, "manifest_pull_count" = %s, "last_manifest_pull_date" = %s WHERE ("manifestpullstatistics"."id" = %s)', [9, 'sha256:3614ca5eacf0a3a1bcc361c939202a974b4902b9334ff36eb29ffe9011aaad83', 1, datetime.datetime(2025, 11, 4, 9, 7, 25), 4]) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,100 [79] [DEBUG] [peewee] ('INSERT INTO "manifestpullstatistics" ("repository_id", "manifest_digest", "manifest_pull_count", "last_manifest_pull_date") VALUES (%s, %s, %s, %s) RETURNING "manifestpullstatistics"."id"', [9, 'sha256:2c5e2045f35086c019e80c86880fd5b7c7a619878b59e3b7592711e1781df51a', 1, datetime.datetime(2025, 11, 4, 9, 7, 25)]) pullstatsredisflushworker stdout | 2025-11-04 09:07:32,102 [79] [INFO] [__main__] RedisFlushWorker: Successfully updated 2/2 manifest statistics pullstatsredisflushworker stdout | 2025-11-04 09:07:32,103 [79] [DEBUG] [__main__] RedisFlushWorker: Successfully deleted 2 Redis keys pullstatsredisflushworker stdout | 2025-11-04 09:07:32,104 [79] [INFO] [__main__] RedisFlushWorker: Successfully processed 2 keys (1 tag updates, 2 manifest updates) in 0.02s pullstatsredisflushworker stdout | 2025-11-04 09:07:32,104 [79] [DEBUG] [data.database] Disconnecting from database. pullstatsredisflushworker stdout | 2025-11-04 09:07:32,104 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:07:32,361 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:32,561 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:07:32,561 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:33.561829+00:00 (in 0.999866 seconds) namespacegcworker stdout | 2025-11-04 09:07:32,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:32 GMT)" (scheduled at 2025-11-04 09:07:32.561493+00:00) namespacegcworker stdout | 2025-11-04 09:07:32,563 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:07:32,570 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), True, datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), 0, 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:07:32,571 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), True, datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 7, 32, 562303), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:07:32,572 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:07:32,573 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:12:32 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:07:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:07:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:53.560131+00:00 (in 19.997870 seconds) namespacegcworker stdout | 2025-11-04 09:07:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" (scheduled at 2025-11-04 09:07:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:07:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:07:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:07:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:07:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:06.569529+00:00 (in 33.001012 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:07:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" (scheduled at 2025-11-04 09:07:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:07:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:07:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 33, 568839), True, datetime.datetime(2025, 11, 4, 9, 7, 33, 568839), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:07:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:07:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:07:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:07:33,959 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:07:34,062 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:07:35,148 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:07:35,643 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:07:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:07:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:07.077358+00:00 (in 29.999519 seconds) autopruneworker stdout | 2025-11-04 09:07:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:07 GMT)" (scheduled at 2025-11-04 09:07:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:07:37,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243657082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:07:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:07:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:07:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:07:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:55.661677+00:00 (in 17.997761 seconds) gcworker stdout | 2025-11-04 09:07:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:07 GMT)" (scheduled at 2025-11-04 09:07:37.663502+00:00) gcworker stdout | 2025-11-04 09:07:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:07:37,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246957669, None, 1, 0]) gcworker stdout | 2025-11-04 09:07:37,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:37,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:47.766253+00:00 (in 9.999587 seconds) notificationworker stdout | 2025-11-04 09:07:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:47 GMT)" (scheduled at 2025-11-04 09:07:37.766253+00:00) notificationworker stdout | 2025-11-04 09:07:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 37, 766991), True, datetime.datetime(2025, 11, 4, 9, 7, 37, 766991), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:07:38,158 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:07:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:12.482065+00:00 (in 34.000967 seconds) exportactionlogsworker stdout | 2025-11-04 09:07:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:38 GMT)" (scheduled at 2025-11-04 09:07:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:07:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:07:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 38, 481483), True, datetime.datetime(2025, 11, 4, 9, 7, 38, 481483), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:07:38,488 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:07:38,488 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:07:38,489 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:07:39,264 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:07:39,543 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:47.679968+00:00 (in 8.003222 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:49 GMT)" (scheduled at 2025-11-04 09:07:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 39, 677076), True, datetime.datetime(2025, 11, 4, 9, 7, 39, 677076), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:49 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:07:39,955 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:07:39,958 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:40,066 [261] [DEBUG] [app] Starting request: urn:request:4e8750e8-5e7d-4607-85c5-33434289dc56 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:40,070 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:40,072 [250] [DEBUG] [app] Starting request: urn:request:f8bca55a-4074-45e0-b0ee-367eaf863404 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:40,073 [250] [DEBUG] [app] Ending request: urn:request:f8bca55a-4074-45e0-b0ee-367eaf863404 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f8bca55a-4074-45e0-b0ee-367eaf863404', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:40,073 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:07:40,073 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:40,074 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:40,076 [257] [DEBUG] [app] Starting request: urn:request:3b9688e9-3f2f-41fb-962b-7b4fa77f9632 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:40,076 [257] [DEBUG] [app] Ending request: urn:request:3b9688e9-3f2f-41fb-962b-7b4fa77f9632 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3b9688e9-3f2f-41fb-962b-7b4fa77f9632', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:40,076 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:40,076 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:40,077 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:40,077 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:40,077 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:40,082 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:40,082 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:40,087 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:40,088 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:40,089 [261] [DEBUG] [app] Ending request: urn:request:4e8750e8-5e7d-4607-85c5-33434289dc56 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4e8750e8-5e7d-4607-85c5-33434289dc56', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:40,089 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:40,089 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:07:40,122 [261] [DEBUG] [app] Starting request: urn:request:865f19be-b2fb-412e-b364-8c4d5251884a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:07:40,124 [260] [DEBUG] [app] Starting request: urn:request:1a1d7989-5144-4dca-bd24-0a7118f10b7d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:40,124 [260] [DEBUG] [app] Ending request: urn:request:1a1d7989-5144-4dca-bd24-0a7118f10b7d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1a1d7989-5144-4dca-bd24-0a7118f10b7d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:40,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:40,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:40,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:40,128 [254] [DEBUG] [app] Starting request: urn:request:0b6aeda1-5130-4f74-9cbb-d81a55979c9f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:40,129 [254] [DEBUG] [app] Ending request: urn:request:0b6aeda1-5130-4f74-9cbb-d81a55979c9f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0b6aeda1-5130-4f74-9cbb-d81a55979c9f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:40,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:07:40,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:07:40,130 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:40,130 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:40,130 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:40,134 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:40,134 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:40,138 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:40,139 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:40,140 [261] [DEBUG] [app] Ending request: urn:request:865f19be-b2fb-412e-b364-8c4d5251884a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:865f19be-b2fb-412e-b364-8c4d5251884a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:40,141 [261] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) gunicorn-web stdout | 2025-11-04 09:07:40,141 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:07:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:07:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:06.265965+00:00 (in 26.084768 seconds) repositorygcworker stdout | 2025-11-04 09:07:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:40 GMT)" (scheduled at 2025-11-04 09:07:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:07:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:07:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 40, 181580), True, datetime.datetime(2025, 11, 4, 9, 7, 40, 181580), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:07:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:07:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:07:40,189 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:07:40,762 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:07:41,055 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:07:41,354 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:41,974 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:41,978 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:41,978 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:07:42,461 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:42,860 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:43,252 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:07:44,055 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:44,479 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:44,479 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:44,480 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:44,537 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:07:44,539 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:07:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:07:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:14.980431+00:00 (in 29.999528 seconds) buildlogsarchiver stdout | 2025-11-04 09:07:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:14 GMT)" (scheduled at 2025-11-04 09:07:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:07:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 7, 44, 981183), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:07:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:07:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:07:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:07:45,964 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,050 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,068 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,072 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,078 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,077 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,080 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,080 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:07:46,083 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:49.676299+00:00 (in 1.995877 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:47 GMT)" (scheduled at 2025-11-04 09:07:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:07:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:07:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:47,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:57.766253+00:00 (in 9.999191 seconds) notificationworker stdout | 2025-11-04 09:07:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:57 GMT)" (scheduled at 2025-11-04 09:07:47.766253+00:00) notificationworker stdout | 2025-11-04 09:07:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 47, 767387), True, datetime.datetime(2025, 11, 4, 9, 7, 47, 767387), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:47,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:47,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:47,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:07:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:07:59.676299+00:00 (in 9.999495 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:59 GMT)" (scheduled at 2025-11-04 09:07:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 49, 677144), True, datetime.datetime(2025, 11, 4, 9, 7, 49, 677144), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:07:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:07:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:07:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:01.707296+00:00 (in 12.000408 seconds) securityworker stdout | 2025-11-04 09:07:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:19 GMT)" (scheduled at 2025-11-04 09:07:49.706478+00:00) securityworker stdout | 2025-11-04 09:07:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:07:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:07:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:07:49,709 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:49,715 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:07:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:49,715 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,716 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 15, 25]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:07:49,717 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:07:49,717 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:07:49,717 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 34, 44]) securityworker stdout | 2025-11-04 09:07:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:07:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:07:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:07:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-25 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [util.migrate.allocator] Right range 15-25 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stdout | 2025-11-04 09:07:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 3, 13]) securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 4 total holes securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 4 total holes securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 3-13 securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:07:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 3-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 13 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 13 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-34 securityworker stdout | 2025-11-04 09:07:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-34 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 25, 35]) securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stdout | 2025-11-04 09:07:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-44 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-44 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Left range 15-44 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:07:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 44, 49]) securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-44 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-44 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 13-15 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-15 securityworker stdout | 2025-11-04 09:07:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-15 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-15 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:07:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 13, 23]) securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 23-13 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-13 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 12, 22]) securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stdout | 2025-11-04 09:07:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-49 securityworker stdout | 2025-11-04 09:07:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 27, 37]) securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-27 securityworker stdout | 2025-11-04 09:07:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-27 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 22, 32]) securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-22 securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-22 securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,732 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-37 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-22 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Left range 12-37 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 38, 48]) securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Right range 12-37 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stdout | 2025-11-04 09:07:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 1, 11]) securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Left range 12-37 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stdout | 2025-11-04 09:07:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stdout | 2025-11-04 09:07:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 37, 47]) securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-37 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-37 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-48 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Right range 12-48 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-12 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 12-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-12 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:07:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 11, 21]) securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 12-48 securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 12-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 48-49 securityworker stdout | 2025-11-04 09:07:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:07:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 48-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:07:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 48, 49]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:07:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 48 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 49-48 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-48 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 21, 31]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:07:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:07:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stdout | 2025-11-04 09:07:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 34, 44]) securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:07:49,746 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stdout | 2025-11-04 09:07:49,746 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:07:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-34 securityworker stdout | 2025-11-04 09:07:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-34 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:07:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 31, 41]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-31 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-31 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-44 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-31 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Left range 21-44 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:07:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:07:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 44, 49]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-44 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-44 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-44 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-21 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-21 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:07:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:07:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 10, 20]) securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-21 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stdout | 2025-11-04 09:07:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-21 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:07:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 1, 11]) securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 20-21 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 20-21 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-21 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-21 securityworker stdout | 2025-11-04 09:07:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:07:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 2, 49, 709649), 20, 30]) securityworker stdout | 2025-11-04 09:07:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 30 securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 30 securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 30-20 securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:07:49,755 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:07:49,755 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] Total range: 30-20 securityworker stderr | 2025-11-04 09:07:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:07:50,867 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:00.086435+00:00 (in 7.925982 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:52 GMT)" (scheduled at 2025-11-04 09:07:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:07:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:07:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:07:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:33.561829+00:00 (in 40.001075 seconds) namespacegcworker stdout | 2025-11-04 09:07:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:53 GMT)" (scheduled at 2025-11-04 09:07:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:07:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:07:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 53, 561075), True, datetime.datetime(2025, 11, 4, 9, 7, 53, 561075), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:07:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:07:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:07:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:08:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:07:55,065 [261] [DEBUG] [app] Starting request: urn:request:db946c75-16fb-4489-b9cb-69a040165d85 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:07:55,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:07:55,068 [260] [DEBUG] [app] Starting request: urn:request:d58aed00-7eb7-42a3-afbf-0b05577d7761 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:55,068 [260] [DEBUG] [app] Ending request: urn:request:d58aed00-7eb7-42a3-afbf-0b05577d7761 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d58aed00-7eb7-42a3-afbf-0b05577d7761', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:55,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:55,069 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:55,070 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:55,071 [261] [DEBUG] [app] Starting request: urn:request:46ce3a3b-af17-4e8a-ae85-0ca6f47ba440 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:55,072 [261] [DEBUG] [app] Ending request: urn:request:46ce3a3b-af17-4e8a-ae85-0ca6f47ba440 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:46ce3a3b-af17-4e8a-ae85-0ca6f47ba440', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:55,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:55,072 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:55,073 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:55,073 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:55,073 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:55,077 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:55,078 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:55,082 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:55,083 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:55,084 [261] [DEBUG] [app] Ending request: urn:request:db946c75-16fb-4489-b9cb-69a040165d85 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:db946c75-16fb-4489-b9cb-69a040165d85', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:55,084 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:55,084 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:07:55,123 [261] [DEBUG] [app] Starting request: urn:request:81454ca7-c474-46c4-b51a-52e63c693bb8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:07:55,127 [260] [DEBUG] [app] Starting request: urn:request:cc3d3211-f935-4948-a54b-0fcb37b31206 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:07:55,127 [260] [DEBUG] [app] Ending request: urn:request:cc3d3211-f935-4948-a54b-0fcb37b31206 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:cc3d3211-f935-4948-a54b-0fcb37b31206', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:07:55,128 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:07:55,128 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:55,129 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:07:55,131 [254] [DEBUG] [app] Starting request: urn:request:f417570a-1f14-4ecf-a9a0-5a4b890a05f3 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:07:55,131 [254] [DEBUG] [app] Ending request: urn:request:f417570a-1f14-4ecf-a9a0-5a4b890a05f3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f417570a-1f14-4ecf-a9a0-5a4b890a05f3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:07:55,131 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:07:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:07:55,132 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:07:55,132 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:07:55,132 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:07:55,132 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:07:55,137 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:07:55,137 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:07:55,141 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:07:55,142 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:07:55,143 [261] [DEBUG] [app] Ending request: urn:request:81454ca7-c474-46c4-b51a-52e63c693bb8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:81454ca7-c474-46c4-b51a-52e63c693bb8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:07:55,143 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:07:55,144 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:07:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:07:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.022) notificationworker stdout | 2025-11-04 09:07:55,578 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:07:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:07:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:07.663502+00:00 (in 12.001424 seconds) gcworker stdout | 2025-11-04 09:07:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:25 GMT)" (scheduled at 2025-11-04 09:07:55.661677+00:00) gcworker stdout | 2025-11-04 09:07:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037675662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:07:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:07:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:07:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:07:55,966 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:07:56,202 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:07:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:07:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:07:56,586 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:07:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:07:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) notificationworker stdout | 2025-11-04 09:07:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:07:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:07.766253+00:00 (in 9.999569 seconds) notificationworker stdout | 2025-11-04 09:07:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:07 GMT)" (scheduled at 2025-11-04 09:07:57.766253+00:00) notificationworker stdout | 2025-11-04 09:07:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:07:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 57, 766970), True, datetime.datetime(2025, 11, 4, 9, 7, 57, 766970), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:07:57,782 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:07:57,782 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:07:57,782 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:07:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:07:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:09.676299+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:07:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:09 GMT)" (scheduled at 2025-11-04 09:07:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:07:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:07:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 7, 59, 677449), True, datetime.datetime(2025, 11, 4, 9, 7, 59, 677449), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:07:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:07:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:07:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:07:59,776 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:07:59,877 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:52.159714+00:00 (in 52.072847 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:00 GMT)" (scheduled at 2025-11-04 09:08:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,093 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:08:00,093 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:00 GMT)" executed successfully builder stdout | 2025-11-04 09:08:00,169 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:08:00,976 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:08:01,171 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:19.706478+00:00 (in 17.998717 seconds) securityworker stdout | 2025-11-04 09:08:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:31 GMT)" (scheduled at 2025-11-04 09:08:01.707296+00:00) securityworker stdout | 2025-11-04 09:08:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:01,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:01,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 1, 711658), 1, 49]) securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 1, 711658), 1, 49]) securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:08:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:08:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:32.081579+00:00 (in 29.999581 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:08:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:32 GMT)" (scheduled at 2025-11-04 09:08:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:08:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:08:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:08:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:08:02,374 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:08:03,973 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:08:04,075 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:08:05,161 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:08:05,656 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:08:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:08:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:40.180734+00:00 (in 33.914156 seconds) repositorygcworker stdout | 2025-11-04 09:08:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:06 GMT)" (scheduled at 2025-11-04 09:08:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:08:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:08:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:08:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:08:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:33.568037+00:00 (in 26.998088 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:08:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:06 GMT)" (scheduled at 2025-11-04 09:08:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:08:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:08:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:08:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:08:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:37.077358+00:00 (in 29.999531 seconds) autopruneworker stdout | 2025-11-04 09:08:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:37 GMT)" (scheduled at 2025-11-04 09:08:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:08:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243687081, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:08:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:08:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:08:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:08:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:25.661677+00:00 (in 17.997755 seconds) gcworker stdout | 2025-11-04 09:08:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:37 GMT)" (scheduled at 2025-11-04 09:08:07.663502+00:00) gcworker stdout | 2025-11-04 09:08:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:08:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762246987671, None, 1, 0]) gcworker stdout | 2025-11-04 09:08:07,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:07,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:17.766253+00:00 (in 9.999537 seconds) notificationworker stdout | 2025-11-04 09:08:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:17 GMT)" (scheduled at 2025-11-04 09:08:07.766253+00:00) notificationworker stdout | 2025-11-04 09:08:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 7, 767135), True, datetime.datetime(2025, 11, 4, 9, 8, 7, 767135), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:08:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:08:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:07.770322+00:00 (in 59.999505 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:08:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:07 GMT)" (scheduled at 2025-11-04 09:08:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:08:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:08:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:08:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:08:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:08:08,171 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:08:09,277 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:09,556 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:19.676299+00:00 (in 9.999583 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:19 GMT)" (scheduled at 2025-11-04 09:08:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 9, 677034), True, datetime.datetime(2025, 11, 4, 9, 8, 9, 677034), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:19 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:09,971 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:08:09,969 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:10,066 [254] [DEBUG] [app] Starting request: urn:request:b2f4d99e-39e5-4cde-a2cf-c4cecdcab32a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:08:10,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:08:10,069 [250] [DEBUG] [app] Starting request: urn:request:3f917bee-5baf-4b44-ace5-e9629e52514a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:10,070 [250] [DEBUG] [app] Ending request: urn:request:3f917bee-5baf-4b44-ace5-e9629e52514a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3f917bee-5baf-4b44-ace5-e9629e52514a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:08:10,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:10,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:10,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:10,072 [254] [DEBUG] [app] Starting request: urn:request:ca59a127-b982-4635-960b-63d377e81cb3 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:10,073 [254] [DEBUG] [app] Ending request: urn:request:ca59a127-b982-4635-960b-63d377e81cb3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ca59a127-b982-4635-960b-63d377e81cb3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.004 159 0.003) gunicorn-web stdout | 2025-11-04 09:08:10,076 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:10,076 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:10,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:10,077 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:10,077 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:10,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:10,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:10,085 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:10,086 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:10,086 [254] [DEBUG] [app] Ending request: urn:request:b2f4d99e-39e5-4cde-a2cf-c4cecdcab32a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b2f4d99e-39e5-4cde-a2cf-c4cecdcab32a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:10,087 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:10,087 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) gunicorn-web stdout | 2025-11-04 09:08:10,122 [254] [DEBUG] [app] Starting request: urn:request:293c7e45-dbfc-4a53-ac07-9c3b0571482c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:08:10,124 [250] [DEBUG] [app] Starting request: urn:request:5a65120e-0a54-46fe-b22b-bd93c86102f7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:10,124 [250] [DEBUG] [app] Ending request: urn:request:5a65120e-0a54-46fe-b22b-bd93c86102f7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5a65120e-0a54-46fe-b22b-bd93c86102f7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:10,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:10,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:08:10,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:10,127 [254] [DEBUG] [app] Starting request: urn:request:6ec8203f-1c67-474f-835b-3ffe531d4b3e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:10,127 [254] [DEBUG] [app] Ending request: urn:request:6ec8203f-1c67-474f-835b-3ffe531d4b3e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6ec8203f-1c67-474f-835b-3ffe531d4b3e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:10,127 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:10,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:10,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:10,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:10,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:10,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:10,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:10,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:10,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:10,137 [254] [DEBUG] [app] Ending request: urn:request:293c7e45-dbfc-4a53-ac07-9c3b0571482c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:293c7e45-dbfc-4a53-ac07-9c3b0571482c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:10,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:10,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.016 118 0.017) quotaregistrysizeworker stdout | 2025-11-04 09:08:10,776 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:08:11,068 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:08:11,366 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:11,988 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:11,992 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:11,993 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:08:12,473 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:08:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:38.480592+00:00 (in 25.997813 seconds) exportactionlogsworker stdout | 2025-11-04 09:08:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:12 GMT)" (scheduled at 2025-11-04 09:08:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:08:12,483 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:08:12,483 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:12 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:08:12,874 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:13,266 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:14,068 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:14,493 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:14,499 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:14,502 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:14,551 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:14,554 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:08:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:08:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:44.980431+00:00 (in 29.999544 seconds) buildlogsarchiver stdout | 2025-11-04 09:08:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:44 GMT)" (scheduled at 2025-11-04 09:08:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:08:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 8, 14, 981198), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:08:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:08:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:08:15,983 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,065 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,092 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,092 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,095 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,096 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,104 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,106 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:16,106 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:17,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:27.766253+00:00 (in 9.999087 seconds) notificationworker stdout | 2025-11-04 09:08:17,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:27 GMT)" (scheduled at 2025-11-04 09:08:17.766253+00:00) notificationworker stdout | 2025-11-04 09:08:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 17, 767535), True, datetime.datetime(2025, 11, 4, 9, 8, 17, 767535), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:08:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:29.676299+00:00 (in 9.999588 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:29 GMT)" (scheduled at 2025-11-04 09:08:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 19, 677045), True, datetime.datetime(2025, 11, 4, 9, 8, 19, 677045), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:31.707296+00:00 (in 12.000360 seconds) securityworker stdout | 2025-11-04 09:08:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:49 GMT)" (scheduled at 2025-11-04 09:08:19.706478+00:00) securityworker stdout | 2025-11-04 09:08:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:08:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:08:20,885 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:25,066 [261] [DEBUG] [app] Starting request: urn:request:43ff109c-d6cf-4f4d-98ed-dc509735d3e8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:08:25,067 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:08:25,069 [260] [DEBUG] [app] Starting request: urn:request:3b7ac554-712d-4227-a50c-d5f96b5467bf (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:25,069 [260] [DEBUG] [app] Ending request: urn:request:3b7ac554-712d-4227-a50c-d5f96b5467bf (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3b7ac554-712d-4227-a50c-d5f96b5467bf', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:08:25,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:25,070 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:25,071 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:25,072 [254] [DEBUG] [app] Starting request: urn:request:6bc87866-026c-49f3-a484-a150c74c8bd0 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:25,073 [254] [DEBUG] [app] Ending request: urn:request:6bc87866-026c-49f3-a484-a150c74c8bd0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6bc87866-026c-49f3-a484-a150c74c8bd0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:25,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:08:25,073 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:25,074 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:25,074 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:25,074 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:25,079 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:25,079 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:25,083 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:25,084 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:25,085 [261] [DEBUG] [app] Ending request: urn:request:43ff109c-d6cf-4f4d-98ed-dc509735d3e8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:43ff109c-d6cf-4f4d-98ed-dc509735d3e8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:25,085 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:25,085 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.021) gunicorn-web stdout | 2025-11-04 09:08:25,123 [261] [DEBUG] [app] Starting request: urn:request:23754594-b505-4a9f-bf9d-56f36566f728 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:08:25,124 [250] [DEBUG] [app] Starting request: urn:request:e33906b4-19d8-4925-9134-6c19782f0041 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:25,125 [250] [DEBUG] [app] Ending request: urn:request:e33906b4-19d8-4925-9134-6c19782f0041 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e33906b4-19d8-4925-9134-6c19782f0041', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:25,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:25,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:25,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:25,127 [254] [DEBUG] [app] Starting request: urn:request:efb1e064-46fd-4d2f-ac8b-c51a877c1d45 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:25,128 [254] [DEBUG] [app] Ending request: urn:request:efb1e064-46fd-4d2f-ac8b-c51a877c1d45 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:efb1e064-46fd-4d2f-ac8b-c51a877c1d45', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:25,128 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:25,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:25,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:25,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:25,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:25,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:25,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:25,136 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:25,137 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:25,138 [261] [DEBUG] [app] Ending request: urn:request:23754594-b505-4a9f-bf9d-56f36566f728 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:23754594-b505-4a9f-bf9d-56f36566f728', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:25,138 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:25,138 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.016 118 0.017) notificationworker stdout | 2025-11-04 09:08:25,591 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:37.663502+00:00 (in 12.001381 seconds) gcworker stdout | 2025-11-04 09:08:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:55 GMT)" (scheduled at 2025-11-04 09:08:25.661677+00:00) gcworker stdout | 2025-11-04 09:08:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037705662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:08:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:08:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:08:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:25,980 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:08:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:27.768179+00:00 (in 0.001504 seconds) notificationworker stdout | 2025-11-04 09:08:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:37 GMT)" (scheduled at 2025-11-04 09:08:27.766253+00:00) notificationworker stdout | 2025-11-04 09:08:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 27, 767018), True, datetime.datetime(2025, 11, 4, 9, 8, 27, 767018), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:37.766253+00:00 (in 9.997742 seconds) notificationworker stdout | 2025-11-04 09:08:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:27 GMT)" (scheduled at 2025-11-04 09:08:27.768179+00:00) notificationworker stdout | 2025-11-04 09:08:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:08:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:39.676299+00:00 (in 9.999595 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:39 GMT)" (scheduled at 2025-11-04 09:08:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 29, 677034), True, datetime.datetime(2025, 11, 4, 9, 8, 29, 677034), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:08:29,790 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:08:29,892 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:08:30,182 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:08:30,990 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:08:31,186 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:08:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:49.706478+00:00 (in 17.998734 seconds) securityworker stdout | 2025-11-04 09:08:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:01 GMT)" (scheduled at 2025-11-04 09:08:31.707296+00:00) securityworker stdout | 2025-11-04 09:08:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:31,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:31,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:31,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 31, 712099), 1, 49]) securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 31, 712099), 1, 49]) securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:08:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:08:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:08:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:02.081579+00:00 (in 29.999559 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:08:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:02 GMT)" (scheduled at 2025-11-04 09:08:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:08:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:08:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:08:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:08:32,387 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:08:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:53.560131+00:00 (in 19.997869 seconds) namespacegcworker stdout | 2025-11-04 09:08:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" (scheduled at 2025-11-04 09:08:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:08:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:08:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:08:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:08:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:06.569529+00:00 (in 33.001063 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:08:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" (scheduled at 2025-11-04 09:08:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:08:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:08:33,570 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 33, 568846), True, datetime.datetime(2025, 11, 4, 9, 8, 33, 568846), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:08:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:08:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:08:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:08:33,994 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:08:34,088 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:08:35,175 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:08:35,676 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:08:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:08:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:07.077358+00:00 (in 29.999508 seconds) autopruneworker stdout | 2025-11-04 09:08:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:07 GMT)" (scheduled at 2025-11-04 09:08:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:08:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243717081, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:08:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:08:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:08:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:08:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:55.661677+00:00 (in 17.997682 seconds) gcworker stdout | 2025-11-04 09:08:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:07 GMT)" (scheduled at 2025-11-04 09:08:37.663502+00:00) gcworker stdout | 2025-11-04 09:08:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:08:37,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247017670, None, 1, 0]) gcworker stdout | 2025-11-04 09:08:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:37,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:47.766253+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:08:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:47 GMT)" (scheduled at 2025-11-04 09:08:37.766253+00:00) notificationworker stdout | 2025-11-04 09:08:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 37, 766944), True, datetime.datetime(2025, 11, 4, 9, 8, 37, 766944), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:08:38,184 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:08:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:12.482065+00:00 (in 34.001049 seconds) exportactionlogsworker stdout | 2025-11-04 09:08:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:38 GMT)" (scheduled at 2025-11-04 09:08:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:08:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:08:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 38, 481819), True, datetime.datetime(2025, 11, 4, 9, 8, 38, 481819), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:08:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:08:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:08:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:08:39,291 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:08:39,570 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:47.679968+00:00 (in 8.003211 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:49 GMT)" (scheduled at 2025-11-04 09:08:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 39, 677096), True, datetime.datetime(2025, 11, 4, 9, 8, 39, 677096), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:39,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:39,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:39,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:49 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:39,984 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:08:39,988 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:40,065 [254] [DEBUG] [app] Starting request: urn:request:275f3183-ec95-4039-aec7-0ccf5cc99bd3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:08:40,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:08:40,070 [260] [DEBUG] [app] Starting request: urn:request:64ebdbbf-6db1-47aa-92b9-309abdb3aeff (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:40,070 [260] [DEBUG] [app] Ending request: urn:request:64ebdbbf-6db1-47aa-92b9-309abdb3aeff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:64ebdbbf-6db1-47aa-92b9-309abdb3aeff', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:40,071 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:40,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:40,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:40,073 [258] [DEBUG] [app] Starting request: urn:request:0bffb5e7-6e45-409f-87f9-756542dbb1a0 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:40,073 [258] [DEBUG] [app] Ending request: urn:request:0bffb5e7-6e45-409f-87f9-756542dbb1a0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0bffb5e7-6e45-409f-87f9-756542dbb1a0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:40,074 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:40,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:40,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:40,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:40,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:40,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:40,083 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:40,084 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:40,085 [254] [DEBUG] [app] Ending request: urn:request:275f3183-ec95-4039-aec7-0ccf5cc99bd3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:275f3183-ec95-4039-aec7-0ccf5cc99bd3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:40,085 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:40,086 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gunicorn-web stdout | 2025-11-04 09:08:40,122 [254] [DEBUG] [app] Starting request: urn:request:220d769f-4235-4a55-8309-baa14462cf0b (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:08:40,123 [250] [DEBUG] [app] Starting request: urn:request:c18da366-5ea7-4554-a504-045e6269eba4 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:40,124 [250] [DEBUG] [app] Ending request: urn:request:c18da366-5ea7-4554-a504-045e6269eba4 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c18da366-5ea7-4554-a504-045e6269eba4', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:40,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:40,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:40,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:40,128 [258] [DEBUG] [app] Starting request: urn:request:e6156cee-05b3-470d-8ea4-c79678742e7d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:40,128 [258] [DEBUG] [app] Ending request: urn:request:e6156cee-05b3-470d-8ea4-c79678742e7d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e6156cee-05b3-470d-8ea4-c79678742e7d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:40,128 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:40,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:40,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:40,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:40,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:40,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:40,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:40,137 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:40,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:40,139 [254] [DEBUG] [app] Ending request: urn:request:220d769f-4235-4a55-8309-baa14462cf0b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:220d769f-4235-4a55-8309-baa14462cf0b', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:40,139 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:08:40,140 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.018) repositorygcworker stdout | 2025-11-04 09:08:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:08:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:06.265965+00:00 (in 26.084806 seconds) repositorygcworker stdout | 2025-11-04 09:08:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:40 GMT)" (scheduled at 2025-11-04 09:08:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:08:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:08:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 40, 181526), True, datetime.datetime(2025, 11, 4, 9, 8, 40, 181526), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:08:40,187 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:08:40,187 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:08:40,187 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:08:40,789 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:08:41,082 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:08:41,380 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:42,007 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:42,010 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:42,013 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:08:42,487 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:42,888 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:43,281 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:08:44,081 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:44,508 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:44,514 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:44,522 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:44,565 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:08:44,570 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:08:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:08:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:14.980431+00:00 (in 29.999574 seconds) buildlogsarchiver stdout | 2025-11-04 09:08:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:14 GMT)" (scheduled at 2025-11-04 09:08:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:08:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 8, 44, 981164), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:08:44,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:08:44,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:08:44,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:08:46,001 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,080 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,109 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,111 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,116 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,117 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,124 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,129 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:08:46,132 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:49.676299+00:00 (in 1.995834 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:47 GMT)" (scheduled at 2025-11-04 09:08:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:08:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:08:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:47,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:57.766253+00:00 (in 9.999165 seconds) notificationworker stdout | 2025-11-04 09:08:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:57 GMT)" (scheduled at 2025-11-04 09:08:47.766253+00:00) notificationworker stdout | 2025-11-04 09:08:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 47, 767447), True, datetime.datetime(2025, 11, 4, 9, 8, 47, 767447), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:47,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:47,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:47,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:08:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:08:59.676299+00:00 (in 9.999520 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:59 GMT)" (scheduled at 2025-11-04 09:08:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 49, 677131), True, datetime.datetime(2025, 11, 4, 9, 8, 49, 677131), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:49,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:49,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:49,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:08:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:08:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:08:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:01.707296+00:00 (in 12.000375 seconds) securityworker stdout | 2025-11-04 09:08:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:19 GMT)" (scheduled at 2025-11-04 09:08:49.706478+00:00) securityworker stdout | 2025-11-04 09:08:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:08:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:08:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:08:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:08:49,715 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,716 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,716 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:49,716 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:08:49,717 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 11, 21]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stdout | 2025-11-04 09:08:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,722 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stdout | 2025-11-04 09:08:49,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-49 securityworker stdout | 2025-11-04 09:08:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:08:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 25, 35]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-25 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-25 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:08:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:08:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 21, 31]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 35 securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 35-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 35-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:08:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:08:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 36, 46]) securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 35-49 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 35-49 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Left range 36-46 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:08:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Left range 36-46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stdout | 2025-11-04 09:08:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 46, 49]) securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 36-46 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Already merged with block 36-46 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 36 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 36-46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 36-46 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 36 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 35-36 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-36 securityworker stdout | 2025-11-04 09:08:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 35-36 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-36 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 35, 45]) securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 45 securityworker stdout | 2025-11-04 09:08:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 45-35 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 45 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 45-35 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:08:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 31, 41]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:08:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:08:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-31 securityworker stdout | 2025-11-04 09:08:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-31 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:08:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 16, 26]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-31 securityworker stdout | 2025-11-04 09:08:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-31 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:08:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 26, 36]) securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-41 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Left range 16-41 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:08:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:08:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 41, 49]) securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-41 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-41 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-41 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 1-16 securityworker stdout | 2025-11-04 09:08:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-16 securityworker stdout | 2025-11-04 09:08:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 6 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-16 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-16 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 6 securityworker stdout | 2025-11-04 09:08:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 1, 11]) securityworker stdout | 2025-11-04 09:08:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:08:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:08:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:08:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 11-16 securityworker stdout | 2025-11-04 09:08:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-16 securityworker stdout | 2025-11-04 09:08:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-16 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-16 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:08:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 11, 21]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:08:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:08:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:08:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:08:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:08:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 33, 43]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:08:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-33 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-33 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:08:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 22, 32]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:08:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:08:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 4, 14]) securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-22 securityworker stdout | 2025-11-04 09:08:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-22 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:08:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 14, 24]) securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-32 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Right range 4-32 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:08:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 4-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:08:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 1, 11]) securityworker stdout | 2025-11-04 09:08:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:08:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:08:49,754 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-32 securityworker stdout | 2025-11-04 09:08:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:08:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 32, 42]) securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 43 securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:08:49,756 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 43-49 securityworker stdout | 2025-11-04 09:08:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:08:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stdout | 2025-11-04 09:08:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 43-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 3, 49, 710271), 43, 49]) securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 43 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] Total range: 49-43 securityworker stdout | 2025-11-04 09:08:49,758 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:49,759 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-43 securityworker stderr | 2025-11-04 09:08:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:08:49,759 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:08:50,898 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:00.086435+00:00 (in 7.926298 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:52 GMT)" (scheduled at 2025-11-04 09:08:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,160 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:08:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:08:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:08:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:33.561829+00:00 (in 40.000978 seconds) namespacegcworker stdout | 2025-11-04 09:08:53,561 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:53 GMT)" (scheduled at 2025-11-04 09:08:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:08:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:08:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 53, 561208), True, datetime.datetime(2025, 11, 4, 9, 8, 53, 561208), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:08:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:08:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:08:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:09:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:08:55,065 [254] [DEBUG] [app] Starting request: urn:request:eebe00a8-b0f2-4f50-8c36-7483c51bd33e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:08:55,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:08:55,068 [250] [DEBUG] [app] Starting request: urn:request:77fc3907-53a8-481a-b715-45c46a3e0150 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:55,069 [250] [DEBUG] [app] Ending request: urn:request:77fc3907-53a8-481a-b715-45c46a3e0150 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77fc3907-53a8-481a-b715-45c46a3e0150', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:08:55,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:08:55,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:55,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:55,071 [261] [DEBUG] [app] Starting request: urn:request:73c70177-da42-4230-b1e2-c82bfda98a46 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:55,072 [261] [DEBUG] [app] Ending request: urn:request:73c70177-da42-4230-b1e2-c82bfda98a46 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:73c70177-da42-4230-b1e2-c82bfda98a46', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:08:55,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:55,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:55,073 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:55,073 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:55,073 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:55,078 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:55,078 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:55,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:55,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:55,084 [254] [DEBUG] [app] Ending request: urn:request:eebe00a8-b0f2-4f50-8c36-7483c51bd33e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:eebe00a8-b0f2-4f50-8c36-7483c51bd33e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:55,084 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:08:55,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:08:55,122 [254] [DEBUG] [app] Starting request: urn:request:d95a1005-213a-4d78-9414-0163cc13a476 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:08:55,123 [250] [DEBUG] [app] Starting request: urn:request:ca243006-15f3-4b91-a3bf-f05bd7fd7d9d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:08:55,124 [250] [DEBUG] [app] Ending request: urn:request:ca243006-15f3-4b91-a3bf-f05bd7fd7d9d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ca243006-15f3-4b91-a3bf-f05bd7fd7d9d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:08:55,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:55,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:08:55,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:08:55,127 [261] [DEBUG] [app] Starting request: urn:request:b071f57b-aef4-4497-87d7-d34cc9998f96 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:08:55,127 [261] [DEBUG] [app] Ending request: urn:request:b071f57b-aef4-4497-87d7-d34cc9998f96 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b071f57b-aef4-4497-87d7-d34cc9998f96', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:08:55,128 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:08:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:08:55,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:08:55,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:08:55,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:08:55,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:08:55,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:08:55,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:08:55,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:08:55,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:08:55,138 [254] [DEBUG] [app] Ending request: urn:request:d95a1005-213a-4d78-9414-0163cc13a476 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d95a1005-213a-4d78-9414-0163cc13a476', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:08:55,138 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:08:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.017) gunicorn-web stdout | 2025-11-04 09:08:55,139 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:08:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" notificationworker stdout | 2025-11-04 09:08:55,603 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:08:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:08:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:07.663502+00:00 (in 12.001389 seconds) gcworker stdout | 2025-11-04 09:08:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:25 GMT)" (scheduled at 2025-11-04 09:08:55.661677+00:00) gcworker stdout | 2025-11-04 09:08:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037735662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:08:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:08:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:08:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:08:55,994 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:08:56,203 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:08:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:08:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:08:56,585 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:08:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:08:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) notificationworker stdout | 2025-11-04 09:08:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:08:57,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:07.766253+00:00 (in 9.999103 seconds) notificationworker stdout | 2025-11-04 09:08:57,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:07 GMT)" (scheduled at 2025-11-04 09:08:57.766253+00:00) notificationworker stdout | 2025-11-04 09:08:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:08:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 57, 767529), True, datetime.datetime(2025, 11, 4, 9, 8, 57, 767529), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:08:57,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:08:57,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:08:57,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:08:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:08:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:09.676299+00:00 (in 9.999425 seconds) proxycacheblobworker stdout | 2025-11-04 09:08:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:09 GMT)" (scheduled at 2025-11-04 09:08:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:08:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:08:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 8, 59, 677262), True, datetime.datetime(2025, 11, 4, 9, 8, 59, 677262), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:08:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:08:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:08:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:08:59,803 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:08:59,906 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:52.159714+00:00 (in 52.072839 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:00 GMT)" (scheduled at 2025-11-04 09:09:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,093 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:09:00,093 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:00 GMT)" executed successfully builder stdout | 2025-11-04 09:09:00,195 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:09:01,003 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:09:01,199 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:19.706478+00:00 (in 17.998727 seconds) securityworker stdout | 2025-11-04 09:09:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:31 GMT)" (scheduled at 2025-11-04 09:09:01.707296+00:00) securityworker stdout | 2025-11-04 09:09:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:01,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:01,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 1, 711789), 1, 49]) securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 1, 711789), 1, 49]) securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:09:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:09:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:32.081579+00:00 (in 29.999573 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:09:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:32 GMT)" (scheduled at 2025-11-04 09:09:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:09:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:09:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:09:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:09:02,401 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:09:04,007 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:09:04,101 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:09:05,188 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:09:05,689 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:09:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:40.180734+00:00 (in 33.914350 seconds) repositorygcworker stdout | 2025-11-04 09:09:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:06 GMT)" (scheduled at 2025-11-04 09:09:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:09:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:09:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:09:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:09:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:33.568037+00:00 (in 26.998090 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:09:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:06 GMT)" (scheduled at 2025-11-04 09:09:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:09:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:09:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:09:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:09:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:37.077358+00:00 (in 29.999526 seconds) autopruneworker stdout | 2025-11-04 09:09:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:37 GMT)" (scheduled at 2025-11-04 09:09:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:09:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243747082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:09:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:09:07,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:09:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:09:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:25.661677+00:00 (in 17.997737 seconds) gcworker stdout | 2025-11-04 09:09:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:37 GMT)" (scheduled at 2025-11-04 09:09:07.663502+00:00) gcworker stdout | 2025-11-04 09:09:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:09:07,669 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247047669, None, 1, 0]) gcworker stdout | 2025-11-04 09:09:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:17.766253+00:00 (in 9.999525 seconds) notificationworker stdout | 2025-11-04 09:09:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:17 GMT)" (scheduled at 2025-11-04 09:09:07.766253+00:00) notificationworker stdout | 2025-11-04 09:09:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 7, 767093), True, datetime.datetime(2025, 11, 4, 9, 9, 7, 767093), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:09:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:09:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:07.770322+00:00 (in 59.999559 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:09:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:07 GMT)" (scheduled at 2025-11-04 09:09:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:09:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:09:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:09:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:09:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:09:08,198 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:09:09,304 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:09,584 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:19.676299+00:00 (in 9.999538 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:19 GMT)" (scheduled at 2025-11-04 09:09:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 9, 677078), True, datetime.datetime(2025, 11, 4, 9, 9, 9, 677078), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:19 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:09,997 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:09:10,001 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:10,065 [258] [DEBUG] [app] Starting request: urn:request:f291cab9-6a63-4cd8-b64f-12f1d221abe0 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:10,080 [258] [DEBUG] [urllib3.connectionpool] Starting new HTTP connection (1): localhost:8080 gunicorn-registry stdout | 2025-11-04 09:09:10,082 [250] [DEBUG] [app] Starting request: urn:request:1b7f0574-90e5-4f55-9686-62f1f4365344 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:10,083 [250] [DEBUG] [app] Ending request: urn:request:1b7f0574-90e5-4f55-9686-62f1f4365344 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:1b7f0574-90e5-4f55-9686-62f1f4365344', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:10,083 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:10,083 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:10,084 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:10,086 [254] [DEBUG] [app] Starting request: urn:request:571e1314-b148-4a7e-9372-353b6d7290ee (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:10,086 [254] [DEBUG] [app] Ending request: urn:request:571e1314-b148-4a7e-9372-353b6d7290ee (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:571e1314-b148-4a7e-9372-353b6d7290ee', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:10,087 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:10,087 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:10,088 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:10,088 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:10,089 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:10,093 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:10,093 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:10,098 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:10,099 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:10,100 [258] [DEBUG] [app] Ending request: urn:request:f291cab9-6a63-4cd8-b64f-12f1d221abe0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f291cab9-6a63-4cd8-b64f-12f1d221abe0', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:10,101 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:10,101 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.036 118 0.036) gunicorn-web stdout | 2025-11-04 09:09:10,123 [258] [DEBUG] [app] Starting request: urn:request:0b8f529a-7b40-4f81-bc9e-4a60f77e4aad (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:09:10,126 [260] [DEBUG] [app] Starting request: urn:request:bfb65ff5-075e-47f8-a028-48ad1a6c267b (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:10,126 [260] [DEBUG] [app] Ending request: urn:request:bfb65ff5-075e-47f8-a028-48ad1a6c267b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bfb65ff5-075e-47f8-a028-48ad1a6c267b', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:10,126 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:10,127 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:09:10,127 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:10,129 [258] [DEBUG] [app] Starting request: urn:request:0b7456f0-c3f0-42d3-bd29-941453b12d0f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:10,129 [258] [DEBUG] [app] Ending request: urn:request:0b7456f0-c3f0-42d3-bd29-941453b12d0f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0b7456f0-c3f0-42d3-bd29-941453b12d0f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:10,130 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:10,130 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:10,130 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:10,130 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:10,130 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:10,134 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:10,134 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:10,137 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:10,138 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:10,139 [258] [DEBUG] [app] Ending request: urn:request:0b8f529a-7b40-4f81-bc9e-4a60f77e4aad (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0b8f529a-7b40-4f81-bc9e-4a60f77e4aad', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:10,139 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:10,139 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.018) quotaregistrysizeworker stdout | 2025-11-04 09:09:10,803 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:09:11,095 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:09:11,394 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:12,016 [260] [DEBUG] [app] Starting request: urn:request:8d2867c3-0348-44f5-a4d8-af2850a83b3b (/v2/) {'X-Forwarded-For': '183.241.154.170, 10.129.4.13'} gunicorn-registry stdout | 2025-11-04 09:09:12,016 [260] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {} gunicorn-registry stdout | 2025-11-04 09:09:12,016 [260] [DEBUG] [auth.registry_jwt_auth] No auth header. gunicorn-registry stdout | 2025-11-04 09:09:12,016 [260] [DEBUG] [app] Ending request: urn:request:8d2867c3-0348-44f5-a4d8-af2850a83b3b (/v2/) {'endpoint': 'v2.v2_support_enabled', 'request_id': 'urn:request:8d2867c3-0348-44f5-a4d8-af2850a83b3b', 'remote_addr': '10.129.4.13', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/', 'path': '/v2/', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:09:12,017 [260] [INFO] [gunicorn.access] 10.129.4.13 - - [04/Nov/2025:09:09:12 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.34.3 (github.com/containers/image)" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:09:12 +0000] "GET /v2/ HTTP/1.1" 401 4 "-" "containers/5.34.3 (github.com/containers/image)" (0.002 574 0.002) gunicorn-secscan stdout | 2025-11-04 09:09:12,022 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:12,027 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:12,028 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:09:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:38.480592+00:00 (in 25.998089 seconds) exportactionlogsworker stdout | 2025-11-04 09:09:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:12 GMT)" (scheduled at 2025-11-04 09:09:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:09:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:09:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:09:12,501 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:12,902 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:13,294 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:13,462 [259] [DEBUG] [app] Starting request: urn:request:204ecbf2-a8e2-42d0-92fe-2ff5bcfb1244 (/v2/quayorg/repo1/manifests/ubuntu) {'X-Forwarded-For': '183.241.154.170, 10.129.4.13'} gunicorn-registry stdout | 2025-11-04 09:09:13,462 [259] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'manifest_ref': 'ubuntu', 'namespace_name': 'quayorg', 'repo_name': 'repo1'} gunicorn-registry stdout | 2025-11-04 09:09:13,462 [259] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:13,463 [259] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:13,465 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 9, 13, 464004), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 9, 13, 464036), 'quay', None, datetime.datetime(2025, 11, 3, 9, 9, 13, 464061), datetime.datetime(2025, 10, 28, 9, 9, 13, 464078), None]) gunicorn-registry stdout | 2025-11-04 09:09:13,465 [259] [DEBUG] [peewee.pool] No connection available in pool. gunicorn-registry stdout | 2025-11-04 09:09:13,469 [259] [DEBUG] [peewee.pool] Created new connection 140331977079232. gunicorn-registry stdout | 2025-11-04 09:09:13,484 [259] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:09:13,484 [259] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: quay gunicorn-registry stdout | 2025-11-04 09:09:13,484 [259] [DEBUG] [auth.registry_jwt_auth] Identity changed to quay gunicorn-registry stdout | 2025-11-04 09:09:13,484 [259] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'user', 'entity_reference': 'd98f031a-b876-4d0a-af2a-2a4bad8a6cd7', 'kind': 'user', 'user': 'quay', 'com.apostille.roots': {'quayorg/repo1': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:09:13,484 [259] [DEBUG] [endpoints.v2] Checking permission for repo: quayorg/repo1 gunicorn-registry stdout | 2025-11-04 09:09:13,485 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['d98f031a-b876-4d0a-af2a-2a4bad8a6cd7', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,487 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['quayorg', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,489 [259] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['quayorg', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:09:13,489 [259] [DEBUG] [data.cache.cache_key] Loading repository lookup from cache_key: repository_lookup_quayorg_repo1_ubuntu gunicorn-registry stdout | 2025-11-04 09:09:13,489 [259] [DEBUG] [data.cache.impl] Checking cache for key repository_lookup_quayorg_repo1_ubuntu gunicorn-registry stdout | 2025-11-04 09:09:13,489 [259] [DEBUG] [data.cache.impl] Found no result in cache for key repository_lookup_quayorg_repo1_ubuntu; calling loader gunicorn-registry stdout | 2025-11-04 09:09:13,491 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['quayorg', 'repo1', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,493 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,494 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,494 [259] [DEBUG] [data.cache.impl] Got loaded result for key repository_lookup_quayorg_repo1_ubuntu: {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}} gunicorn-registry stdout | 2025-11-04 09:09:13,494 [259] [DEBUG] [data.cache.impl] Caching loaded result for key repository_lookup_quayorg_repo1_ubuntu with expiration {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:09:13,495 [259] [DEBUG] [data.cache.impl] Cached loaded result for key repository_lookup_quayorg_repo1_ubuntu with expiration {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:09:13,495 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,496 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['public', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,498 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."repository_id", "t1"."manifest_id", "t1"."lifetime_start_ms", "t1"."lifetime_end_ms", "t1"."immutable", "t1"."hidden", "t1"."reversion", "t1"."tag_kind_id", "t1"."linked_tag_id", "t2"."id", "t2"."repository_id", "t2"."digest", "t2"."media_type_id", "t2"."manifest_bytes", "t2"."config_media_type", "t2"."layers_compressed_size", "t2"."subject", "t2"."subject_backfilled", "t2"."artifact_type", "t2"."artifact_type_backfilled" FROM "tag" AS "t1" INNER JOIN "manifest" AS "t2" ON ("t1"."manifest_id" = "t2"."id") WHERE (((("t1"."repository_id" = %s) AND ("t1"."name" = %s)) AND (("t1"."lifetime_end_ms" IS %s) OR ("t1"."lifetime_end_ms" > %s))) AND ("t1"."hidden" = %s)) LIMIT %s OFFSET %s', [10, 'ubuntu', None, 1762247353497, False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,500 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "mediatype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,502 [259] [DEBUG] [util.audit] Checking publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:09:13,502 [259] [DEBUG] [util.audit] Publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:09:13,503 [259] [DEBUG] [util.audit] Resolving IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:09:13,504 [259] [DEBUG] [util.audit] Resolved IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:09:13,504 [259] [DEBUG] [util.audit] Logging the pull_repo to logs system gunicorn-registry stdout | 2025-11-04 09:09:13,505 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['quayorg', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,506 [259] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "logentrykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,508 [259] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [41, 2, 1, 10, datetime.datetime(2025, 11, 4, 9, 9, 13, 505048), '10.129.4.13', '{"repo": "repo1", "namespace": "quayorg", "user-agent": "containers/5.34.3 (github.com/containers/image)", "tag": "ubuntu", "username": "quay", "resolved_ip": {"provider": "internet", "service": null, "sync_token": "1645662201", "country_iso_code": null, "aws_region": null, "continent": null}}']) gunicorn-registry stdout | 2025-11-04 09:09:13,510 [259] [DEBUG] [util.audit] Track and log of pull_repo complete gunicorn-registry stdout | 2025-11-04 09:09:13,511 [259] [DEBUG] [app] Ending request: urn:request:204ecbf2-a8e2-42d0-92fe-2ff5bcfb1244 (/v2/quayorg/repo1/manifests/ubuntu) {'endpoint': 'v2.fetch_manifest_by_tagname', 'request_id': 'urn:request:204ecbf2-a8e2-42d0-92fe-2ff5bcfb1244', 'remote_addr': '10.129.4.13', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/quayorg/repo1/manifests/ubuntu', 'path': '/v2/quayorg/repo1/manifests/ubuntu', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:09:13,511 [259] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:09:13,511 [259] [DEBUG] [peewee.pool] Returning 140331977079232 to pool. gunicorn-registry stdout | 2025-11-04 09:09:13,512 [259] [INFO] [gunicorn.access] 10.129.4.13 - - [04/Nov/2025:09:09:13 +0000] "GET /v2/quayorg/repo1/manifests/ubuntu HTTP/1.1" 200 1416 "-" "containers/5.34.3 (github.com/containers/image)" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:09:13 +0000] "GET /v2/quayorg/repo1/manifests/ubuntu HTTP/1.1" 200 1416 "-" "containers/5.34.3 (github.com/containers/image)" (0.052 2057 0.051) gunicorn-registry stdout | 2025-11-04 09:09:13,518 [259] [DEBUG] [data.userevent] Published user event docker-cli: {'action': 'pull_repo', 'repository': 'repo1', 'namespace': 'quayorg'} gunicorn-registry stdout | 2025-11-04 09:09:13,520 [259] [DEBUG] [util.pullmetrics] Tracked tag pull: repo_id=10 tag=ubuntu digest=sha256:b6b83d3c331794420340093eb706a6f152d9c1fa51b262d9bf34594887c2c7ac gunicorn-registry stdout | 2025-11-04 09:09:13,778 [250] [DEBUG] [app] Starting request: urn:request:01b68098-9d7e-42f7-a12b-1087fde8299f (/v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8) {'X-Forwarded-For': '183.241.154.170, 10.129.4.13'} gunicorn-registry stdout | 2025-11-04 09:09:13,779 [250] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'manifest_ref': 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'namespace_name': 'quayorg', 'repo_name': 'repo1'} gunicorn-registry stdout | 2025-11-04 09:09:13,779 [250] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:13,779 [250] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:13,782 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name", "t1"."kid", "t1"."service", "t1"."jwk", "t1"."metadata", "t1"."created_date", "t1"."expiration_date", "t1"."rotation_duration", "t1"."approval_id" FROM "servicekey" AS "t1" LEFT OUTER JOIN "servicekeyapproval" AS "t2" ON ("t1"."approval_id" = "t2"."id") WHERE ((((NOT ("t1"."approval_id" IS %s) AND (("t1"."expiration_date" > %s) OR ("t1"."expiration_date" IS %s))) AND ("t1"."service" = %s)) AND (NOT (("t1"."service" = %s) AND ("t1"."expiration_date" <= %s)) OR NOT ((("t1"."service" = %s) AND ("t1"."approval_id" IS %s)) AND ("t1"."created_date" <= %s)))) AND (NOT ("t1"."expiration_date" <= %s) OR ("t1"."expiration_date" IS %s)))', [None, datetime.datetime(2025, 11, 4, 9, 9, 13, 781087), None, 'quay', 'quay', datetime.datetime(2025, 11, 4, 9, 9, 13, 781120), 'quay', None, datetime.datetime(2025, 11, 3, 9, 9, 13, 781144), datetime.datetime(2025, 10, 28, 9, 9, 13, 781162), None]) gunicorn-registry stdout | 2025-11-04 09:09:13,782 [250] [DEBUG] [peewee.pool] No connection available in pool. gunicorn-registry stdout | 2025-11-04 09:09:13,787 [250] [DEBUG] [peewee.pool] Created new connection 140331977079232. gunicorn-registry stdout | 2025-11-04 09:09:13,801 [250] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:09:13,801 [250] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: quay gunicorn-registry stdout | 2025-11-04 09:09:13,802 [250] [DEBUG] [auth.registry_jwt_auth] Identity changed to quay gunicorn-registry stdout | 2025-11-04 09:09:13,802 [250] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'user', 'entity_reference': 'd98f031a-b876-4d0a-af2a-2a4bad8a6cd7', 'kind': 'user', 'user': 'quay', 'com.apostille.roots': {'quayorg/repo1': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:09:13,802 [250] [DEBUG] [endpoints.v2] Checking permission for repo: quayorg/repo1 gunicorn-registry stdout | 2025-11-04 09:09:13,803 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['d98f031a-b876-4d0a-af2a-2a4bad8a6cd7', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,805 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['quayorg', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,806 [250] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['quayorg', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:09:13,806 [250] [DEBUG] [data.cache.cache_key] Loading repository lookup from cache_key: repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 gunicorn-registry stdout | 2025-11-04 09:09:13,806 [250] [DEBUG] [data.cache.impl] Checking cache for key repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 gunicorn-registry stdout | 2025-11-04 09:09:13,807 [250] [DEBUG] [data.cache.impl] Found no result in cache for key repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8; calling loader gunicorn-registry stdout | 2025-11-04 09:09:13,808 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['quayorg', 'repo1', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,810 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,810 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,811 [250] [DEBUG] [data.cache.impl] Got loaded result for key repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8: {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}} gunicorn-registry stdout | 2025-11-04 09:09:13,811 [250] [DEBUG] [data.cache.impl] Caching loaded result for key repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 with expiration {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:09:13,811 [250] [DEBUG] [data.cache.impl] Cached loaded result for key repository_lookup_quayorg_repo1_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 with expiration {'id': 10, 'visibility': {'id': 2, 'name': 'private'}, 'kind': {'id': 1, 'name': 'image'}, 'state': , 'namespace_user': {'stripe_id': None}}: 120s gunicorn-registry stdout | 2025-11-04 09:09:13,812 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,813 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['public', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,813 [250] [DEBUG] [data.cache.impl] Checking cache for key repository_manifest__10_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 gunicorn-registry stdout | 2025-11-04 09:09:13,813 [250] [DEBUG] [data.cache.impl] Found no result in cache for key repository_manifest__10_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8; calling loader gunicorn-registry stdout | 2025-11-04 09:09:13,814 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" INNER JOIN "tag" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t1"."repository_id" = %s) AND ("t1"."digest" = %s)) AND (("t2"."lifetime_end_ms" IS %s) OR ("t2"."lifetime_end_ms" > %s))) LIMIT %s OFFSET %s', [10, 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', None, 1762247353814, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,817 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "mediatype" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,818 [250] [DEBUG] [data.cache.impl] Got loaded result for key repository_manifest__10_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8: {'digest': 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1478,\n "digest": "sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 28378212,\n "digest": "sha256:ed02c6ade914c2962413c1ad2ccc86ed8d1512098f2c87fe7bafa8e1b5293185"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 28378212, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 44, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 10, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}} gunicorn-registry stdout | 2025-11-04 09:09:13,818 [250] [DEBUG] [data.cache.impl] Caching loaded result for key repository_manifest__10_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 with expiration {'digest': 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1478,\n "digest": "sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 28378212,\n "digest": "sha256:ed02c6ade914c2962413c1ad2ccc86ed8d1512098f2c87fe7bafa8e1b5293185"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 28378212, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 44, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 10, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}}: 300s gunicorn-registry stdout | 2025-11-04 09:09:13,818 [250] [DEBUG] [data.cache.impl] Cached loaded result for key repository_manifest__10_sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 with expiration {'digest': 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'internal_manifest_bytes': '{\n "schemaVersion": 2,\n "mediaType": "application/vnd.docker.distribution.manifest.v2+json",\n "config": {\n "mediaType": "application/vnd.docker.container.image.v1+json",\n "size": 1478,\n "digest": "sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf"\n },\n "layers": [\n {\n "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",\n "size": 28378212,\n "digest": "sha256:ed02c6ade914c2962413c1ad2ccc86ed8d1512098f2c87fe7bafa8e1b5293185"\n }\n ]\n}', 'media_type': 'application/vnd.docker.distribution.manifest.v2+json', '_layers_compressed_size': 28378212, 'config_media_type': 'application/vnd.docker.container.image.v1+json', 'db_id': 44, 'inputs': {'legacy_id_handler': None, 'legacy_image_row': None, 'repository': {'db_id': 10, 'inputs': {'kind': None, 'is_public': None, 'namespace_name': None, 'repo_name': None, 'is_free_namespace': None, 'state': None}}, 'legacy_image_handler': None}}: 300s gunicorn-registry stdout | 2025-11-04 09:09:13,818 [250] [DEBUG] [util.audit] Checking publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:09:13,819 [250] [DEBUG] [util.audit] Publishing pull_repo to the user events system gunicorn-registry stdout | 2025-11-04 09:09:13,819 [250] [DEBUG] [util.audit] Logging the pull_repo to analytics engine gunicorn-registry stdout | 2025-11-04 09:09:13,819 [250] [DEBUG] [util.audit] Resolving IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:09:13,821 [250] [DEBUG] [util.audit] Resolved IP address 10.129.4.13 gunicorn-registry stdout | 2025-11-04 09:09:13,821 [250] [DEBUG] [util.audit] Logging the pull_repo to logs system gunicorn-registry stdout | 2025-11-04 09:09:13,821 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE ("t1"."username" = %s) LIMIT %s OFFSET %s', ['quayorg', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:13,822 [250] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "logentrykind" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:13,825 [250] [DEBUG] [peewee] ('INSERT INTO "logentry3" ("kind_id", "account_id", "performer_id", "repository_id", "datetime", "ip", "metadata_json") VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING "logentry3"."id"', [41, 2, 1, 10, datetime.datetime(2025, 11, 4, 9, 9, 13, 821209), '10.129.4.13', '{"repo": "repo1", "namespace": "quayorg", "user-agent": "containers/5.34.3 (github.com/containers/image)", "manifest_digest": "sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8", "username": "quay", "resolved_ip": {"provider": "internet", "service": null, "sync_token": "1645662201", "country_iso_code": null, "aws_region": null, "continent": null}}']) gunicorn-registry stdout | 2025-11-04 09:09:13,827 [250] [DEBUG] [util.audit] Track and log of pull_repo complete gunicorn-registry stdout | 2025-11-04 09:09:13,828 [250] [DEBUG] [app] Ending request: urn:request:01b68098-9d7e-42f7-a12b-1087fde8299f (/v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8) {'endpoint': 'v2.fetch_manifest_by_digest', 'request_id': 'urn:request:01b68098-9d7e-42f7-a12b-1087fde8299f', 'remote_addr': '10.129.4.13', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'path': '/v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:09:13,828 [250] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:09:13,828 [250] [DEBUG] [peewee.pool] Returning 140331977079232 to pool. gunicorn-registry stdout | 2025-11-04 09:09:13,828 [250] [INFO] [gunicorn.access] 10.129.4.13 - - [04/Nov/2025:09:09:13 +0000] "GET /v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 HTTP/1.1" 200 529 "-" "containers/5.34.3 (github.com/containers/image)" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:09:13 +0000] "GET /v2/quayorg/repo1/manifests/sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 HTTP/1.1" 200 529 "-" "containers/5.34.3 (github.com/containers/image)" (0.051 2122 0.052) gunicorn-registry stdout | 2025-11-04 09:09:13,832 [250] [DEBUG] [data.userevent] Published user event docker-cli: {'action': 'pull_repo', 'repository': 'repo1', 'namespace': 'quayorg'} gunicorn-registry stdout | 2025-11-04 09:09:13,833 [250] [DEBUG] [util.pullmetrics] Tracked manifest pull: repo_id=10 digest=sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8 gunicorn-registry stdout | 2025-11-04 09:09:14,092 [255] [DEBUG] [app] Starting request: urn:request:91ee3118-e77c-49e9-b7cd-bd332a800bd0 (/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf) {'X-Forwarded-For': '183.241.154.170, 10.129.4.13'} gunicorn-registry stdout | 2025-11-04 09:09:14,092 [255] [DEBUG] [auth.registry_jwt_auth] Called with params: (), {'digest': 'sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'namespace_name': 'quayorg', 'repo_name': 'repo1'} gunicorn-registry stdout | 2025-11-04 09:09:14,092 [255] [DEBUG] [auth.registry_jwt_auth] Validating auth header: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:14,092 [255] [DEBUG] [util.security.registry_jwt] encoded JWT: eyJhbGciOiJSUzI1NiIsImtpZCI6IllpRTZPa3RFYzJodEg2ZWxQN1lNSVFJNmdqOXBIbzRTR05Nczc0TzRnLTAiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzUzLCJpYXQiOjE3NjIyNDczNTMsImV4cCI6MTc2MjI1MDk1Mywic3ViIjoicXVheSIsImFjY2VzcyI6W3sidHlwZSI6InJlcG9zaXRvcnkiLCJuYW1lIjoicXVheW9yZy9yZXBvMSIsImFjdGlvbnMiOlsicHVsbCJdfV0sImNvbnRleHQiOnsidmVyc2lvbiI6MiwiZW50aXR5X2tpbmQiOiJ1c2VyIiwiZW50aXR5X3JlZmVyZW5jZSI6ImQ5OGYwMzFhLWI4NzYtNGQwYS1hZjJhLTJhNGJhZDhhNmNkNyIsImtpbmQiOiJ1c2VyIiwidXNlciI6InF1YXkiLCJjb20uYXBvc3RpbGxlLnJvb3RzIjp7InF1YXlvcmcvcmVwbzEiOiIkZGlzYWJsZWQifSwiY29tLmFwb3N0aWxsZS5yb290IjoiJGRpc2FibGVkIn19.UD4HIS2jY6u7mApxT-ZKY7nF-eMlu0EGHPL96FkJw5UAElA3ul0uMhm95UoR60S9GHeh8h1mWbnk4Jr6yXVDLM4eQWujf_as0uTvboVJKDtO7MjGXjuBmD3cq7BPOD7BKZq_Td-wk82KZZo9yS-pDCU9EWv54digzdao_aSSUB4BknLyhKTpKdl58wN6SBpn8n236p2z0J1n0RAXNch5KCpnppEnx7NNWbiy4zn000N5ymALrkK3kf1TXbzLK6CzEZhkeFp205RoVppul24LKxjt9vzItXi57Pcy46t6jV0vD1FDXdmOq8QcMoh-LHZBX2FKJaXOM8nvQM7gym2j5A gunicorn-registry stdout | 2025-11-04 09:09:14,093 [255] [DEBUG] [auth.permissions] Identity loaded: gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [auth.permissions] Loaded signed_jwt identity for: quay gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [auth.registry_jwt_auth] Identity changed to quay gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [auth.registry_jwt_auth] Auth context set to {'version': 2, 'entity_kind': 'user', 'entity_reference': 'd98f031a-b876-4d0a-af2a-2a4bad8a6cd7', 'kind': 'user', 'user': 'quay', 'com.apostille.roots': {'quayorg/repo1': '$disabled'}, 'com.apostille.root': '$disabled'} gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [endpoints.v2] Checking permission for repo: quayorg/repo1 gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [data.cache.impl] Checking cache for key geo_restrictions__quayorg gunicorn-registry stdout | 2025-11-04 09:09:14,094 [255] [DEBUG] [data.cache.impl] Found no result in cache for key geo_restrictions__quayorg; calling loader gunicorn-registry stdout | 2025-11-04 09:09:14,095 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."added", "t1"."description", "t1"."unstructured_json", "t1"."restricted_region_iso_code" FROM "namespacegeorestriction" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_id" = "t2"."id") WHERE ("t2"."username" = %s)', ['quayorg']) gunicorn-registry stdout | 2025-11-04 09:09:14,096 [255] [DEBUG] [data.cache.impl] Got loaded result for key geo_restrictions__quayorg: [] gunicorn-registry stdout | 2025-11-04 09:09:14,096 [255] [DEBUG] [data.cache.impl] Caching loaded result for key geo_restrictions__quayorg with expiration []: 240s gunicorn-registry stdout | 2025-11-04 09:09:14,096 [255] [DEBUG] [data.cache.impl] Cached loaded result for key geo_restrictions__quayorg with expiration []: 240s namespacegcworker stdout | 2025-11-04 09:09:14,095 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:14,097 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."username", "t1"."password_hash", "t1"."email", "t1"."verified", "t1"."stripe_id", "t1"."organization", "t1"."robot", "t1"."invoice_email", "t1"."invalid_login_attempts", "t1"."last_invalid_login", "t1"."removed_tag_expiration_s", "t1"."enabled", "t1"."invoice_email_address", "t1"."given_name", "t1"."family_name", "t1"."company", "t1"."location", "t1"."maximum_queued_builds_count", "t1"."creation_date", "t1"."last_accessed" FROM "user" AS "t1" WHERE (("t1"."uuid" = %s) AND ("t1"."organization" = %s)) LIMIT %s OFFSET %s', ['d98f031a-b876-4d0a-af2a-2a4bad8a6cd7', False, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,098 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s', ['quayorg', True, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,099 [255] [DEBUG] [endpoints.decorators] Skipping pull through proxy cache: instance matching query does not exist: gunicorn-registry stdout | SQL: SELECT "t1"."id", "t1"."organization_id", "t1"."creation_date", "t1"."upstream_registry", "t1"."upstream_registry_username", "t1"."upstream_registry_password", "t1"."expiration_s", "t1"."insecure" FROM "proxycacheconfig" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."organization_id" = "t2"."id") WHERE (("t2"."username" = %s) AND ("t2"."organization" = %s)) LIMIT %s OFFSET %s gunicorn-registry stdout | Params: ['quayorg', True, 1, 0] gunicorn-registry stdout | 2025-11-04 09:09:14,099 [255] [DEBUG] [data.cache.impl] Checking cache for key repo_blob__quayorg_repo1_sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf_2 gunicorn-registry stdout | 2025-11-04 09:09:14,099 [255] [DEBUG] [data.cache.impl] Found no result in cache for key repo_blob__quayorg_repo1_sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf_2; calling loader gunicorn-registry stdout | 2025-11-04 09:09:14,100 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state", "t2"."id", "t2"."uuid", "t2"."username", "t2"."password_hash", "t2"."email", "t2"."verified", "t2"."stripe_id", "t2"."organization", "t2"."robot", "t2"."invoice_email", "t2"."invalid_login_attempts", "t2"."last_invalid_login", "t2"."removed_tag_expiration_s", "t2"."enabled", "t2"."invoice_email_address", "t2"."given_name", "t2"."family_name", "t2"."company", "t2"."location", "t2"."maximum_queued_builds_count", "t2"."creation_date", "t2"."last_accessed" FROM "repository" AS "t1" INNER JOIN "user" AS "t2" ON ("t1"."namespace_user_id" = "t2"."id") WHERE ((("t2"."username" = %s) AND ("t1"."name" = %s)) AND ("t1"."state" != %s)) LIMIT %s OFFSET %s', ['quayorg', 'repo1', 3, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,101 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "visibility" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [2, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,102 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "repositorykind" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [1, 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,103 [255] [DEBUG] [peewee] ('SELECT "t1"."uuid" FROM "imagestorage" AS "t1" INNER JOIN "uploadedblob" AS "t2" ON ("t2"."blob_id" = "t1"."id") WHERE (("t2"."repository_id" = %s) AND ("t1"."content_checksum" = %s)) LIMIT %s OFFSET %s', [10, 'sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 1, 0]) gunicorn-registry stdout | 2025-11-04 09:09:14,105 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id", "t2"."id", "t2"."uuid", "t2"."image_size", "t2"."uncompressed_size", "t2"."uploading", "t2"."cas_path", "t2"."content_checksum" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['3ba345dd-d3e5-49bf-92a4-9f3634520db3']) gunicorn-registry stdout | 2025-11-04 09:09:14,106 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "imagestoragelocation" AS "t1"', []) gunicorn-registry stdout | 2025-11-04 09:09:14,107 [255] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."storage_id", "t1"."location_id" FROM "imagestorageplacement" AS "t1" INNER JOIN "imagestorage" AS "t2" ON ("t1"."storage_id" = "t2"."id") WHERE ("t2"."uuid" = %s)', ['3ba345dd-d3e5-49bf-92a4-9f3634520db3']) gunicorn-registry stdout | 2025-11-04 09:09:14,108 [255] [DEBUG] [data.cache.impl] Got loaded result for key repo_blob__quayorg_repo1_sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf_2: {'uuid': '3ba345dd-d3e5-49bf-92a4-9f3634520db3', 'digest': 'sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'compressed_size': 1478, 'uncompressed_size': None, 'uploading': True, 'db_id': 102, 'inputs': {'placements': ['local_us'], 'storage_path': 'sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf'}} gunicorn-registry stdout | 2025-11-04 09:09:14,108 [255] [DEBUG] [data.cache.impl] Caching loaded result for key repo_blob__quayorg_repo1_sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf_2 with expiration {'uuid': '3ba345dd-d3e5-49bf-92a4-9f3634520db3', 'digest': 'sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'compressed_size': 1478, 'uncompressed_size': None, 'uploading': True, 'db_id': 102, 'inputs': {'placements': ['local_us'], 'storage_path': 'sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf'}}: 60s gunicorn-registry stdout | 2025-11-04 09:09:14,108 [255] [DEBUG] [data.cache.impl] Cached loaded result for key repo_blob__quayorg_repo1_sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf_2 with expiration {'uuid': '3ba345dd-d3e5-49bf-92a4-9f3634520db3', 'digest': 'sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'compressed_size': 1478, 'uncompressed_size': None, 'uploading': True, 'db_id': 102, 'inputs': {'placements': ['local_us'], 'storage_path': 'sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf'}}: 60s gunicorn-registry stdout | 2025-11-04 09:09:14,108 [255] [DEBUG] [endpoints.v2.blob] Looking up the direct download URL for path: sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf gunicorn-registry stdout | 2025-11-04 09:09:14,108 [255] [DEBUG] [endpoints.v2.blob] Checking for namespace quayorg gunicorn-registry stdout | 2025-11-04 09:09:14,111 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/endpoints.json gunicorn-registry stdout | 2025-11-04 09:09:14,130 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/sdk-default-configuration.json gunicorn-registry stdout | 2025-11-04 09:09:14,130 [255] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,144 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/service-2.json gunicorn-registry stdout | 2025-11-04 09:09:14,163 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/s3/2006-03-01/endpoint-rule-set-1.json gunicorn-registry stdout | 2025-11-04 09:09:14,167 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/partitions.json gunicorn-registry stdout | 2025-11-04 09:09:14,168 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,168 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7fa196de9a80> gunicorn-registry stdout | 2025-11-04 09:09:14,182 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,184 [255] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) gunicorn-registry stdout | 2025-11-04 09:09:14,187 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/botocore/data/_retry.json gunicorn-registry stdout | 2025-11-04 09:09:14,192 [255] [DEBUG] [botocore.client] Registering retry handlers for service: s3 gunicorn-registry stdout | 2025-11-04 09:09:14,192 [255] [DEBUG] [botocore.utils] Registering S3 region redirector handler gunicorn-registry stdout | 2025-11-04 09:09:14,206 [255] [DEBUG] [botocore.loaders] Loading JSON file: /opt/app-root/lib64/python3.12/site-packages/boto3/data/s3/2006-03-01/resources-1.json gunicorn-registry stdout | 2025-11-04 09:09:14,207 [255] [DEBUG] [botocore.hooks] Event choose-service-name: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,208 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,208 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler ._handler at 0x7fa196de9a80> gunicorn-registry stdout | 2025-11-04 09:09:14,208 [255] [DEBUG] [botocore.hooks] Event creating-client-class.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,209 [255] [DEBUG] [botocore.endpoint] Setting s3 timeout as (60, 60) gunicorn-registry stdout | 2025-11-04 09:09:14,211 [255] [DEBUG] [botocore.client] Registering retry handlers for service: s3 gunicorn-registry stdout | 2025-11-04 09:09:14,211 [255] [DEBUG] [botocore.utils] Registering S3 region redirector handler gunicorn-registry stdout | 2025-11-04 09:09:14,212 [255] [DEBUG] [boto3.resources.factory] Loading s3:s3 gunicorn-registry stdout | 2025-11-04 09:09:14,213 [255] [DEBUG] [boto3.resources.factory] Loading s3:Bucket gunicorn-registry stdout | 2025-11-04 09:09:14,213 [255] [DEBUG] [boto3.resources.model] Renaming Bucket attribute name gunicorn-registry stdout | 2025-11-04 09:09:14,214 [255] [DEBUG] [botocore.hooks] Event creating-resource-class.s3.Bucket: calling handler ._handler at 0x7fa196de9da0> gunicorn-registry stdout | 2025-11-04 09:09:14,214 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,214 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,214 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,214 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,215 [255] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,215 [255] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,215 [255] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:09:14,216 [255] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 gunicorn-registry stdout | 2025-11-04 09:09:14,216 [255] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" gunicorn-registry stdout | 2025-11-04 09:09:14,216 [255] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event before-call.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.endpoint] Making request for OperationModel(name=HeadBucket) with params: {'url_path': '', 'query_string': {}, 'method': 'HEAD', 'headers': {'User-Agent': 'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61'}, 'body': b'', 'auth_path': '/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/', 'url': 'https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'context': {'client_region': 'us-east-1', 'client_config': , 'has_streaming_input': False, 'auth_type': 'v4', 's3_redirect': {'redirected': False, 'bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'params': {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84'}}, 'signing': {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True}}} gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,217 [255] [DEBUG] [botocore.hooks] Event choose-signer.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.hooks] Event before-sign.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.auth] Calculating signature using v4 auth. gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.auth] CanonicalRequest: gunicorn-registry stdout | HEAD gunicorn-registry stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 gunicorn-registry stdout | host:s3.openshift-storage.svc.cluster.local gunicorn-registry stdout | x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | x-amz-date:20251104T090914Z gunicorn-registry stdout | host;x-amz-content-sha256;x-amz-date gunicorn-registry stdout | e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | AWS4-HMAC-SHA256 gunicorn-registry stdout | 20251104T090914Z gunicorn-registry stdout | 20251104/us-east-1/s3/aws4_request gunicorn-registry stdout | 58d6aa0681f73c86c8dcfe2f1f4876f4cfd16638cc5f9f16d5d30bc3f5e15125 gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.auth] Signature: gunicorn-registry stdout | 1b484ce33285ef32a1f90a860441f39e96518c1f9b51071f10c8906a4713196d gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.hooks] Event request-created.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,218 [255] [DEBUG] [botocore.endpoint] Sending http request: gunicorn-registry stdout | 2025-11-04 09:09:14,219 [255] [DEBUG] [botocore.hooks] Event before-send.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,219 [255] [ERROR] [storage.cloud] [OTEL] request {'User-Agent': b'Boto3/1.28.61 md/Botocore#1.31.61 ua/2.0 os/linux#5.14.0-570.60.1.el9_6.x86_64 md/arch#x86_64 lang/python#3.12.9 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.31.61', 'X-Amz-Date': b'20251104T090914Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=e4bs1VRU5uzVgbV0vo0m/20251104/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=1b484ce33285ef32a1f90a860441f39e96518c1f9b51071f10c8906a4713196d', 'amz-sdk-invocation-id': b'2a2cf4b3-f39a-43d2-bd0f-eb2d4a3dd61d', 'amz-sdk-request': b'attempt=1', 'traceparent': '00-0-0-01', 'x-b3-traceid': '0', 'x-b3-spanid': '0', 'x-b3-parentspanid': '0', 'x-b3-sampled': '1'} gunicorn-registry stdout | 2025-11-04 09:09:14,220 [255] [DEBUG] [botocore.httpsession] Certificate path: /opt/app-root/lib64/python3.12/site-packages/certifi/cacert.pem gunicorn-registry stdout | 2025-11-04 09:09:14,220 [255] [DEBUG] [urllib3.connectionpool] Starting new HTTPS connection (1): s3.openshift-storage.svc.cluster.local:443 gunicorn-registry stdout | 2025-11-04 09:09:14,268 [255] [DEBUG] [urllib3.connectionpool] https://s3.openshift-storage.svc.cluster.local:443 "HEAD /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 HTTP/1.1" 200 0 gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.parsers] Response headers: {'Server': 'NooBaa/5.19.4-3154414', 'x-amz-request-id': 'mhkckozz-71ravt-36j', 'x-amz-id-2': 'mhkckozz-71ravt-36j', 'Date': 'Tue, 04 Nov 2025 09:09:14 GMT', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=5'} gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.parsers] Response body: gunicorn-registry stdout | b'' gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.retryhandler] No retry needed. gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.hooks] Event needs-retry.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.hooks] Event after-call.s3.HeadBucket: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,269 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-parameter-build.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-endpoint-resolution.s3: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.regions] Calling endpoint provider with parameters: {'Bucket': 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84', 'Region': 'us-east-1', 'UseFIPS': False, 'UseDualStack': False, 'Endpoint': 'https://s3.openshift-storage.svc.cluster.local:443', 'ForcePathStyle': True, 'Accelerate': False, 'UseGlobalEndpoint': True, 'DisableMultiRegionAccessPoints': False, 'UseArnRegion': True} gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.regions] Endpoint provider result: https://s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84 gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.regions] Selecting from endpoint provider's list of auth schemes: "sigv4". User selected auth scheme is: "None" gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.regions] Selected auth type "v4" as "v4" with signing context params: {'region': 'us-east-1', 'signing_name': 's3', 'disableDoubleEncoding': True} gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event choose-signer.s3.GetObject: calling handler > gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.hooks] Event before-sign.s3.GetObject: calling handler gunicorn-registry stdout | 2025-11-04 09:09:14,270 [255] [DEBUG] [botocore.auth] Calculating signature using hmacv1 auth. gunicorn-registry stdout | 2025-11-04 09:09:14,271 [255] [DEBUG] [botocore.auth] HTTP request method: GET gunicorn-registry stdout | 2025-11-04 09:09:14,271 [255] [DEBUG] [botocore.auth] StringToSign: gunicorn-registry stdout | GET gunicorn-registry stdout | 1762247954 gunicorn-registry stdout | /quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf gunicorn-registry stdout | 2025-11-04 09:09:14,317 [255] [DEBUG] [storage.downloadproxy] Proxying via URL https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/_storage_proxy/ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklrY3RVbGRpVnpCNk5rbFhaSGRUZGxZNE9VVmhTbVl3U1VKUmIyTlhWMjk2UmtRekxVUnpjalZZYUZFaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM016VTBMQ0pwWVhRaU9qRTNOakl5TkRjek5UUXNJbVY0Y0NJNk1UYzJNakkwTnprMU5Dd2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk5aE55OWhOemczTUdaa05EYzRaalF6TnpJNE4ySmxaV1V5TURobVpUVTFOemxqWlRRellqQXpabUZsTWpneU1XUXpPV1kzTjJJek5UQm1OMlJoTlRGaU1XSm1QMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDFpV1c1ak1FWnNaRzVtVUVFek5VdERjSG8wTVdwTGNUSnJhVWtsTTBRbVJYaHdhWEpsY3oweE56WXlNalEzT1RVMElpd2lhRzl6ZENJNkluTXpMbTl3Wlc1emFHbG1kQzF6ZEc5eVlXZGxMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNPalEwTXlJc0luTmphR1Z0WlNJNkltaDBkSEJ6SW4xZExDSmpiMjUwWlhoMElqcDdmWDAuUjhsaUdxeHlrSXdtSDVTMUdBMGpkRVJhdlJSa01IbmJUcmc1bkk2ZTdrX0ZUWG5Wb3RUVzU1c2VkMlVFSjdNbEZoMVhrX044UTNiUXdLYi1nRjZ3ZGMtY05jQVU1ZVRsMk9oNU9BNnBsNVY0ZlAwZnRfbXdHNk82RkJ5NHRpWUtjbFRZUUlqUVk5UC0xX0FqSnAzNmpwVWVhRHlBS2xUWUxUY0NDQ3RwWEhxSTlfMEswTVQtQk9iX1ptbkEtaXdObF9mTmJfdXBNR0N3SWdGalQ3OEJfclVXSlNXbEpoZGVac3VIai1FclhLS3R1MW5OQXppVnBFb2p1c1RGNFUzU19GYUF1SklfMEoxZWxXQlB3VllFbEtMVlRrQ2xhalROSDNKVnJrME45TEZqX3lURXFNOW9EdTk0VDlLaW9OUG8wUkFzMmRCUWgzem1xUDBJdzdzRzFR/https/s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=bYnc0FldnfPA35KCpz41jKq2kiI%3D&Expires=1762247954 gunicorn-registry stdout | 2025-11-04 09:09:14,317 [255] [DEBUG] [endpoints.v2.blob] Returning direct download URL gunicorn-registry stdout | 2025-11-04 09:09:14,317 [255] [DEBUG] [app] Ending request: urn:request:91ee3118-e77c-49e9-b7cd-bd332a800bd0 (/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf) {'endpoint': 'v2.download_blob', 'request_id': 'urn:request:91ee3118-e77c-49e9-b7cd-bd332a800bd0', 'remote_addr': '10.129.4.13', 'http_method': 'GET', 'original_url': 'https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'path': '/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-registry stdout | 2025-11-04 09:09:14,318 [255] [DEBUG] [data.database] Disconnecting from database. gunicorn-registry stdout | 2025-11-04 09:09:14,318 [255] [DEBUG] [peewee.pool] Returning 140331977079232 to pool. gunicorn-registry stdout | 2025-11-04 09:09:14,318 [255] [INFO] [gunicorn.access] 10.129.4.13 - - [04/Nov/2025:09:09:14 +0000] "GET /v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf HTTP/1.1" 302 4171 "-" "containers/5.34.3 (github.com/containers/image)" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:09:14 +0000] "GET /v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf HTTP/1.1" 302 4171 "-" "containers/5.34.3 (github.com/containers/image)" (0.228 1759 0.228) gunicorn-web stdout | 2025-11-04 09:09:14,523 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:14,529 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:14,539 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:14,582 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:14,585 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:09:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:09:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:44.980431+00:00 (in 29.999503 seconds) buildlogsarchiver stdout | 2025-11-04 09:09:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:44 GMT)" (scheduled at 2025-11-04 09:09:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:09:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 9, 14, 981241), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:09:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:09:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:15,104 [254] [DEBUG] [app] Starting request: urn:request:1e3c84e3-1b50-4851-b1ec-83ccdc5b1dc2 (/_storage_proxy_auth) {'X-Forwarded-For': '183.241.154.170'} gunicorn-web stdout | 2025-11-04 09:09:15,104 [254] [DEBUG] [storage.downloadproxy] Got token b'eyJhbGciOiJSUzI1NiIsImtpZCI6IkctUldiVzB6NklXZHdTdlY4OUVhSmYwSUJRb2NXV296RkQzLURzcjVYaFEiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJxdWF5IiwiYXVkIjoicXVheXJlZ2lzdHJ5LXF1YXktcXVheS1lbnRlcnByaXNlLTE1NTA5LmFwcHMucXVheXRlc3QtMTU1MDkucWUuZGV2Y2x1c3Rlci5vcGVuc2hpZnQuY29tIiwibmJmIjoxNzYyMjQ3MzU0LCJpYXQiOjE3NjIyNDczNTQsImV4cCI6MTc2MjI0Nzk1NCwic3ViIjoic3RvcmFnZXByb3h5IiwiYWNjZXNzIjpbeyJ0eXBlIjoic3RvcmFnZXByb3h5IiwidXJpIjoicXVheS1kYXRhc3RvcmUtOWVjOGM2NGYtNjM0ZC00NmNmLWEyOGQtOWM0ZjAyMDA2YTg0L2RhdGFzdG9yYWdlL3JlZ2lzdHJ5L3NoYTI1Ni9hNy9hNzg3MGZkNDc4ZjQzNzI4N2JlZWUyMDhmZTU1NzljZTQzYjAzZmFlMjgyMWQzOWY3N2IzNTBmN2RhNTFiMWJmP0FXU0FjY2Vzc0tleUlkPWU0YnMxVlJVNXV6VmdiVjB2bzBtJlNpZ25hdHVyZT1iWW5jMEZsZG5mUEEzNUtDcHo0MWpLcTJraUklM0QmRXhwaXJlcz0xNzYyMjQ3OTU0IiwiaG9zdCI6InMzLm9wZW5zaGlmdC1zdG9yYWdlLnN2Yy5jbHVzdGVyLmxvY2FsOjQ0MyIsInNjaGVtZSI6Imh0dHBzIn1dLCJjb250ZXh0Ijp7fX0.R8liGqxykIwmH5S1GA0jdERavRRkMHnbTrg5nI6e7k_FTXnVotTW55sed2UEJ7MlFh1Xk_N8Q3bQwKb-gF6wdc-cNcAU5eTl2Oh5OA6pl5V4fP0ft_mwG6O6FBy4tiYKclTYQIjQY9P-1_AjJp36jpUeaDyAKlTYLTcCCCtpXHqI9_0K0MT-BOb_ZmnA-iwNl_fNb_upMGCwIgFjT78B_rUWJSWlJhdeZsuHj-ErXKKtu1nNAziVpEojusTF4U3S_FaAuJI_0J1elWBPwVYElKLVTkClajTNH3JVrk0N9LFj_yTEqM9oDu94T9KioNPo0RAs2dBQh3zmqP0Iw7sG1Q' for storage proxy auth request /_storage_proxy/ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklrY3RVbGRpVnpCNk5rbFhaSGRUZGxZNE9VVmhTbVl3U1VKUmIyTlhWMjk2UmtRekxVUnpjalZZYUZFaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM016VTBMQ0pwWVhRaU9qRTNOakl5TkRjek5UUXNJbVY0Y0NJNk1UYzJNakkwTnprMU5Dd2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk5aE55OWhOemczTUdaa05EYzRaalF6TnpJNE4ySmxaV1V5TURobVpUVTFOemxqWlRRellqQXpabUZsTWpneU1XUXpPV1kzTjJJek5UQm1OMlJoTlRGaU1XSm1QMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDFpV1c1ak1FWnNaRzVtVUVFek5VdERjSG8wTVdwTGNUSnJhVWtsTTBRbVJYaHdhWEpsY3oweE56WXlNalEzT1RVMElpd2lhRzl6ZENJNkluTXpMbTl3Wlc1emFHbG1kQzF6ZEc5eVlXZGxMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNPalEwTXlJc0luTmphR1Z0WlNJNkltaDBkSEJ6SW4xZExDSmpiMjUwWlhoMElqcDdmWDAuUjhsaUdxeHlrSXdtSDVTMUdBMGpkRVJhdlJSa01IbmJUcmc1bkk2ZTdrX0ZUWG5Wb3RUVzU1c2VkMlVFSjdNbEZoMVhrX044UTNiUXdLYi1nRjZ3ZGMtY05jQVU1ZVRsMk9oNU9BNnBsNVY0ZlAwZnRfbXdHNk82RkJ5NHRpWUtjbFRZUUlqUVk5UC0xX0FqSnAzNmpwVWVhRHlBS2xUWUxUY0NDQ3RwWEhxSTlfMEswTVQtQk9iX1ptbkEtaXdObF9mTmJfdXBNR0N3SWdGalQ3OEJfclVXSlNXbEpoZGVac3VIai1FclhLS3R1MW5OQXppVnBFb2p1c1RGNFUzU19GYUF1SklfMEoxZWxXQlB3VllFbEtMVlRrQ2xhalROSDNKVnJrME45TEZqX3lURXFNOW9EdTk0VDlLaW9OUG8wUkFzMmRCUWgzem1xUDBJdzdzRzFR/https/s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=bYnc0FldnfPA35KCpz41jKq2kiI%3D&Expires=1762247954 with parts ['ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklrY3RVbGRpVnpCNk5rbFhaSGRUZGxZNE9VVmhTbVl3U1VKUmIyTlhWMjk2UmtRekxVUnpjalZZYUZFaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM016VTBMQ0pwWVhRaU9qRTNOakl5TkRjek5UUXNJbVY0Y0NJNk1UYzJNakkwTnprMU5Dd2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk5aE55OWhOemczTUdaa05EYzRaalF6TnpJNE4ySmxaV1V5TURobVpUVTFOemxqWlRRellqQXpabUZsTWpneU1XUXpPV1kzTjJJek5UQm1OMlJoTlRGaU1XSm1QMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDFpV1c1ak1FWnNaRzVtVUVFek5VdERjSG8wTVdwTGNUSnJhVWtsTTBRbVJYaHdhWEpsY3oweE56WXlNalEzT1RVMElpd2lhRzl6ZENJNkluTXpMbTl3Wlc1emFHbG1kQzF6ZEc5eVlXZGxMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNPalEwTXlJc0luTmphR1Z0WlNJNkltaDBkSEJ6SW4xZExDSmpiMjUwWlhoMElqcDdmWDAuUjhsaUdxeHlrSXdtSDVTMUdBMGpkRVJhdlJSa01IbmJUcmc1bkk2ZTdrX0ZUWG5Wb3RUVzU1c2VkMlVFSjdNbEZoMVhrX044UTNiUXdLYi1nRjZ3ZGMtY05jQVU1ZVRsMk9oNU9BNnBsNVY0ZlAwZnRfbXdHNk82RkJ5NHRpWUtjbFRZUUlqUVk5UC0xX0FqSnAzNmpwVWVhRHlBS2xUWUxUY0NDQ3RwWEhxSTlfMEswTVQtQk9iX1ptbkEtaXdObF9mTmJfdXBNR0N3SWdGalQ3OEJfclVXSlNXbEpoZGVac3VIai1FclhLS3R1MW5OQXppVnBFb2p1c1RGNFUzU19GYUF1SklfMEoxZWxXQlB3VllFbEtMVlRrQ2xhalROSDNKVnJrME45TEZqX3lURXFNOW9EdTk0VDlLaW9OUG8wUkFzMmRCUWgzem1xUDBJdzdzRzFR', 'https', 's3.openshift-storage.svc.cluster.local:443', 'quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=bYnc0FldnfPA35KCpz41jKq2kiI%3D&Expires=1762247954'] gunicorn-web stdout | 2025-11-04 09:09:15,240 [254] [DEBUG] [app] Ending request: urn:request:1e3c84e3-1b50-4851-b1ec-83ccdc5b1dc2 (/_storage_proxy_auth) {'endpoint': '_storage_proxy_auth', 'request_id': 'urn:request:1e3c84e3-1b50-4851-b1ec-83ccdc5b1dc2', 'remote_addr': '183.241.154.170', 'http_method': 'GET', 'original_url': 'https://web_app_server/_storage_proxy_auth', 'path': '/_storage_proxy_auth', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'containers/5.34.3 (github.com/containers/image)'} gunicorn-web stdout | 2025-11-04 09:09:15,241 [254] [INFO] [gunicorn.access] 183.241.154.170 - - [04/Nov/2025:09:09:15 +0000] "GET /_storage_proxy_auth HTTP/1.0" 200 2 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf" "containers/5.34.3 (github.com/containers/image)" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:09:15 +0000] "GET /_storage_proxy/ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklrY3RVbGRpVnpCNk5rbFhaSGRUZGxZNE9VVmhTbVl3U1VKUmIyTlhWMjk2UmtRekxVUnpjalZZYUZFaUxDSjBlWEFpT2lKS1YxUWlmUS5leUpwYzNNaU9pSnhkV0Y1SWl3aVlYVmtJam9pY1hWaGVYSmxaMmx6ZEhKNUxYRjFZWGt0Y1hWaGVTMWxiblJsY25CeWFYTmxMVEUxTlRBNUxtRndjSE11Y1hWaGVYUmxjM1F0TVRVMU1Ea3VjV1V1WkdWMlkyeDFjM1JsY2k1dmNHVnVjMmhwWm5RdVkyOXRJaXdpYm1KbUlqb3hOell5TWpRM016VTBMQ0pwWVhRaU9qRTNOakl5TkRjek5UUXNJbVY0Y0NJNk1UYzJNakkwTnprMU5Dd2ljM1ZpSWpvaWMzUnZjbUZuWlhCeWIzaDVJaXdpWVdOalpYTnpJanBiZXlKMGVYQmxJam9pYzNSdmNtRm5aWEJ5YjNoNUlpd2lkWEpwSWpvaWNYVmhlUzFrWVhSaGMzUnZjbVV0T1dWak9HTTJOR1l0TmpNMFpDMDBObU5tTFdFeU9HUXRPV00wWmpBeU1EQTJZVGcwTDJSaGRHRnpkRzl5WVdkbEwzSmxaMmx6ZEhKNUwzTm9ZVEkxTmk5aE55OWhOemczTUdaa05EYzRaalF6TnpJNE4ySmxaV1V5TURobVpUVTFOemxqWlRRellqQXpabUZsTWpneU1XUXpPV1kzTjJJek5UQm1OMlJoTlRGaU1XSm1QMEZYVTBGalkyVnpjMHRsZVVsa1BXVTBZbk14VmxKVk5YVjZWbWRpVmpCMmJ6QnRKbE5wWjI1aGRIVnlaVDFpV1c1ak1FWnNaRzVtVUVFek5VdERjSG8wTVdwTGNUSnJhVWtsTTBRbVJYaHdhWEpsY3oweE56WXlNalEzT1RVMElpd2lhRzl6ZENJNkluTXpMbTl3Wlc1emFHbG1kQzF6ZEc5eVlXZGxMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNPalEwTXlJc0luTmphR1Z0WlNJNkltaDBkSEJ6SW4xZExDSmpiMjUwWlhoMElqcDdmWDAuUjhsaUdxeHlrSXdtSDVTMUdBMGpkRVJhdlJSa01IbmJUcmc1bkk2ZTdrX0ZUWG5Wb3RUVzU1c2VkMlVFSjdNbEZoMVhrX044UTNiUXdLYi1nRjZ3ZGMtY05jQVU1ZVRsMk9oNU9BNnBsNVY0ZlAwZnRfbXdHNk82RkJ5NHRpWUtjbFRZUUlqUVk5UC0xX0FqSnAzNmpwVWVhRHlBS2xUWUxUY0NDQ3RwWEhxSTlfMEswTVQtQk9iX1ptbkEtaXdObF9mTmJfdXBNR0N3SWdGalQ3OEJfclVXSlNXbEpoZGVac3VIai1FclhLS3R1MW5OQXppVnBFb2p1c1RGNFUzU19GYUF1SklfMEoxZWxXQlB3VllFbEtMVlRrQ2xhalROSDNKVnJrME45TEZqX3lURXFNOW9EdTk0VDlLaW9OUG8wUkFzMmRCUWgzem1xUDBJdzdzRzFR/https/s3.openshift-storage.svc.cluster.local:443/quay-datastore-9ec8c64f-634d-46cf-a28d-9c4f02006a84/datastorage/registry/sha256/a7/a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf?AWSAccessKeyId=e4bs1VRU5uzVgbV0vo0m&Signature=bYnc0FldnfPA35KCpz41jKq2kiI%3D&Expires=1762247954 HTTP/1.1" 200 673 "https://quayregistry-quay-quay-enterprise-15509.apps.quaytest-15509.qe.devcluster.openshift.com/v2/quayorg/repo1/blobs/sha256:a7870fd478f437287beee208fe5579ce43b03fae2821d39f77b350f7da51b1bf" "containers/5.34.3 (github.com/containers/image)" (0.177 3754 0.039) gunicorn-registry stdout | 2025-11-04 09:09:16,019 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,095 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,126 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,128 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,141 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,144 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,145 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,152 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:16,147 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:27.766253+00:00 (in 9.999547 seconds) notificationworker stdout | 2025-11-04 09:09:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:27 GMT)" (scheduled at 2025-11-04 09:09:17.766253+00:00) notificationworker stdout | 2025-11-04 09:09:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 17, 767037), True, datetime.datetime(2025, 11, 4, 9, 9, 17, 767037), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:17,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:17,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:17,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:09:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:29.676299+00:00 (in 9.999612 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:29 GMT)" (scheduled at 2025-11-04 09:09:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 19, 677125), True, datetime.datetime(2025, 11, 4, 9, 9, 19, 677125), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:31.707296+00:00 (in 12.000370 seconds) securityworker stdout | 2025-11-04 09:09:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:49 GMT)" (scheduled at 2025-11-04 09:09:19.706478+00:00) securityworker stdout | 2025-11-04 09:09:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:09:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:09:20,910 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:25,066 [258] [DEBUG] [app] Starting request: urn:request:c512666a-97e9-4348-b5e5-a89d77f81351 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:25,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:09:25,068 [260] [DEBUG] [app] Starting request: urn:request:c4bac367-17c1-4fb3-ba59-8d429e5abe1e (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:25,069 [260] [DEBUG] [app] Ending request: urn:request:c4bac367-17c1-4fb3-ba59-8d429e5abe1e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c4bac367-17c1-4fb3-ba59-8d429e5abe1e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:09:25,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:25,069 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:25,070 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:25,072 [261] [DEBUG] [app] Starting request: urn:request:693a73a8-9345-4284-bb3c-2aa8e93ba35e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:25,072 [261] [DEBUG] [app] Ending request: urn:request:693a73a8-9345-4284-bb3c-2aa8e93ba35e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:693a73a8-9345-4284-bb3c-2aa8e93ba35e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:25,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:09:25,073 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:25,073 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:25,073 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:25,073 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:25,078 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:25,078 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:25,082 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:25,083 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:25,084 [258] [DEBUG] [app] Ending request: urn:request:c512666a-97e9-4348-b5e5-a89d77f81351 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c512666a-97e9-4348-b5e5-a89d77f81351', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:25,084 [258] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) gunicorn-web stdout | 2025-11-04 09:09:25,085 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:09:25,122 [258] [DEBUG] [app] Starting request: urn:request:d67dc11d-3112-4ee1-a561-9597761f32b1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:09:25,124 [250] [DEBUG] [app] Starting request: urn:request:61e61327-371f-4414-8306-94a229527f72 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:25,125 [250] [DEBUG] [app] Ending request: urn:request:61e61327-371f-4414-8306-94a229527f72 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:61e61327-371f-4414-8306-94a229527f72', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:25,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:09:25,126 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:25,126 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:25,128 [254] [DEBUG] [app] Starting request: urn:request:2e8632f6-41dc-4a76-ba76-ce91129d5eab (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:25,128 [254] [DEBUG] [app] Ending request: urn:request:2e8632f6-41dc-4a76-ba76-ce91129d5eab (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2e8632f6-41dc-4a76-ba76-ce91129d5eab', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:25,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:25,129 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:25,129 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:25,129 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:25,129 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:25,133 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:25,133 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:25,136 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:25,138 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:25,139 [258] [DEBUG] [app] Ending request: urn:request:d67dc11d-3112-4ee1-a561-9597761f32b1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d67dc11d-3112-4ee1-a561-9597761f32b1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:25,139 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:25,139 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) notificationworker stdout | 2025-11-04 09:09:25,615 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:37.663502+00:00 (in 12.001427 seconds) gcworker stdout | 2025-11-04 09:09:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:55 GMT)" (scheduled at 2025-11-04 09:09:25.661677+00:00) gcworker stdout | 2025-11-04 09:09:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037765662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:09:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:09:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:09:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:26,009 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:09:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:27,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:27.768179+00:00 (in 0.001061 seconds) notificationworker stdout | 2025-11-04 09:09:27,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:37 GMT)" (scheduled at 2025-11-04 09:09:27.766253+00:00) notificationworker stdout | 2025-11-04 09:09:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 27, 767577), True, datetime.datetime(2025, 11, 4, 9, 9, 27, 767577), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:37.766253+00:00 (in 9.997417 seconds) notificationworker stdout | 2025-11-04 09:09:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:27 GMT)" (scheduled at 2025-11-04 09:09:27.768179+00:00) notificationworker stdout | 2025-11-04 09:09:27,769 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:09:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:39.676299+00:00 (in 9.999530 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:39 GMT)" (scheduled at 2025-11-04 09:09:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 29, 677093), True, datetime.datetime(2025, 11, 4, 9, 9, 29, 677093), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:29,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:09:29,817 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:09:29,919 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:09:30,207 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:09:31,017 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:09:31,213 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:09:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:49.706478+00:00 (in 17.998801 seconds) securityworker stdout | 2025-11-04 09:09:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:01 GMT)" (scheduled at 2025-11-04 09:09:31.707296+00:00) securityworker stdout | 2025-11-04 09:09:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:31,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:31,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 31, 712639), 1, 49]) securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:31,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 31, 712639), 1, 49]) securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:31,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:09:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:31,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:09:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:09:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:02.081579+00:00 (in 29.999567 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:02 GMT)" (scheduled at 2025-11-04 09:09:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:09:32,085 [79] [INFO] [__main__] RedisFlushWorker: Processing 2 Redis keys pullstatsredisflushworker stdout | 2025-11-04 09:09:32,093 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."tag_name", "t1"."tag_pull_count", "t1"."last_tag_pull_date", "t1"."current_manifest_digest" FROM "tagpullstatistics" AS "t1" WHERE (("t1"."repository_id" = %s) AND ("t1"."tag_name" = %s))', [10, 'ubuntu']) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,095 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,096 [79] [DEBUG] [peewee] ('UPDATE "tagpullstatistics" SET "repository_id" = %s, "tag_name" = %s, "tag_pull_count" = %s, "last_tag_pull_date" = %s, "current_manifest_digest" = %s WHERE ("tagpullstatistics"."id" = %s)', [10, 'ubuntu', 1, datetime.datetime(2025, 11, 4, 9, 9, 13), 'sha256:b6b83d3c331794420340093eb706a6f152d9c1fa51b262d9bf34594887c2c7ac', 1]) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,098 [79] [INFO] [__main__] RedisFlushWorker: Successfully updated 1/1 tag statistics pullstatsredisflushworker stdout | 2025-11-04 09:09:32,098 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."manifest_digest", "t1"."manifest_pull_count", "t1"."last_manifest_pull_date" FROM "manifestpullstatistics" AS "t1" WHERE ((("t1"."repository_id" = %s) AND ("t1"."manifest_digest" = %s)) OR (("t1"."repository_id" = %s) AND ("t1"."manifest_digest" = %s)))', [10, 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 10, 'sha256:b6b83d3c331794420340093eb706a6f152d9c1fa51b262d9bf34594887c2c7ac']) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,099 [79] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_user_id", "t1"."name", "t1"."visibility_id", "t1"."description", "t1"."badge_token", "t1"."kind_id", "t1"."trust_enabled", "t1"."state" FROM "repository" AS "t1" WHERE ("t1"."id" = %s) LIMIT %s OFFSET %s', [10, 1, 0]) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,099 [79] [DEBUG] [peewee] ('UPDATE "manifestpullstatistics" SET "repository_id" = %s, "manifest_digest" = %s, "manifest_pull_count" = %s, "last_manifest_pull_date" = %s WHERE ("manifestpullstatistics"."id" = %s)', [10, 'sha256:b6b83d3c331794420340093eb706a6f152d9c1fa51b262d9bf34594887c2c7ac', 1, datetime.datetime(2025, 11, 4, 9, 9, 13), 3]) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,100 [79] [DEBUG] [peewee] ('INSERT INTO "manifestpullstatistics" ("repository_id", "manifest_digest", "manifest_pull_count", "last_manifest_pull_date") VALUES (%s, %s, %s, %s) RETURNING "manifestpullstatistics"."id"', [10, 'sha256:0f744430d9643a0ec647a4addcac14b1fbb11424be434165c15e2cc7269f70f8', 1, datetime.datetime(2025, 11, 4, 9, 9, 13)]) pullstatsredisflushworker stdout | 2025-11-04 09:09:32,101 [79] [INFO] [__main__] RedisFlushWorker: Successfully updated 2/2 manifest statistics pullstatsredisflushworker stdout | 2025-11-04 09:09:32,103 [79] [DEBUG] [__main__] RedisFlushWorker: Successfully deleted 2 Redis keys pullstatsredisflushworker stdout | 2025-11-04 09:09:32,103 [79] [INFO] [__main__] RedisFlushWorker: Successfully processed 2 keys (1 tag updates, 2 manifest updates) in 0.02s pullstatsredisflushworker stdout | 2025-11-04 09:09:32,103 [79] [DEBUG] [data.database] Disconnecting from database. pullstatsredisflushworker stdout | 2025-11-04 09:09:32,103 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:09:32,414 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:09:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:09:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:53.560131+00:00 (in 19.997809 seconds) namespacegcworker stdout | 2025-11-04 09:09:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" (scheduled at 2025-11-04 09:09:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:09:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:09:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:09:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:09:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:06.569529+00:00 (in 33.001071 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:09:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" (scheduled at 2025-11-04 09:09:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:09:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:09:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 33, 568774), True, datetime.datetime(2025, 11, 4, 9, 9, 33, 568774), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:09:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:09:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:09:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:09:34,019 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:09:34,114 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:09:35,202 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:09:35,702 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:09:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:09:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:07.077358+00:00 (in 29.999525 seconds) autopruneworker stdout | 2025-11-04 09:09:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:07 GMT)" (scheduled at 2025-11-04 09:09:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:09:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243777082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:09:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:09:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:09:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:09:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:55.661677+00:00 (in 17.997763 seconds) gcworker stdout | 2025-11-04 09:09:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:07 GMT)" (scheduled at 2025-11-04 09:09:37.663502+00:00) gcworker stdout | 2025-11-04 09:09:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:09:37,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247077669, None, 1, 0]) gcworker stdout | 2025-11-04 09:09:37,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:37,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:47.766253+00:00 (in 9.999600 seconds) notificationworker stdout | 2025-11-04 09:09:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:47 GMT)" (scheduled at 2025-11-04 09:09:37.766253+00:00) notificationworker stdout | 2025-11-04 09:09:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 37, 766994), True, datetime.datetime(2025, 11, 4, 9, 9, 37, 766994), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:37,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:37,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:37,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:09:38,211 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:09:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:12.482065+00:00 (in 34.001037 seconds) exportactionlogsworker stdout | 2025-11-04 09:09:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:38 GMT)" (scheduled at 2025-11-04 09:09:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:09:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:09:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 38, 481387), True, datetime.datetime(2025, 11, 4, 9, 9, 38, 481387), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:09:38,488 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:09:38,488 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:09:38,488 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:09:39,317 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:09:39,598 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:44.679698+00:00 (in 5.002976 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:49 GMT)" (scheduled at 2025-11-04 09:09:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 39, 677055), True, datetime.datetime(2025, 11, 4, 9, 9, 39, 677055), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:49 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:40,010 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:09:40,013 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:40,065 [258] [DEBUG] [app] Starting request: urn:request:f4828486-a324-425c-b550-4a676677cc2f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:40,069 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:09:40,071 [260] [DEBUG] [app] Starting request: urn:request:d2ef45c7-1572-4271-9286-52775e972612 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:40,071 [260] [DEBUG] [app] Ending request: urn:request:d2ef45c7-1572-4271-9286-52775e972612 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d2ef45c7-1572-4271-9286-52775e972612', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:40,072 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:40,072 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:40,073 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:40,074 [261] [DEBUG] [app] Starting request: urn:request:df0a1a26-ad8f-4b52-b859-770c079dba5d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:40,074 [261] [DEBUG] [app] Ending request: urn:request:df0a1a26-ad8f-4b52-b859-770c079dba5d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:df0a1a26-ad8f-4b52-b859-770c079dba5d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:40,075 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:40,075 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:40,075 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:40,075 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:40,075 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:40,079 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:40,079 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:40,084 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:40,085 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:40,086 [258] [DEBUG] [app] Ending request: urn:request:f4828486-a324-425c-b550-4a676677cc2f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f4828486-a324-425c-b550-4a676677cc2f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:40,086 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:40,086 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) gunicorn-web stdout | 2025-11-04 09:09:40,123 [257] [DEBUG] [app] Starting request: urn:request:11a99bc4-490f-4476-8af8-f8e2518898bb (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:40,124 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:09:40,126 [260] [DEBUG] [app] Starting request: urn:request:8b86dc09-2a39-40e0-9aab-406aae92ac28 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:40,126 [260] [DEBUG] [app] Ending request: urn:request:8b86dc09-2a39-40e0-9aab-406aae92ac28 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8b86dc09-2a39-40e0-9aab-406aae92ac28', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:40,127 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:40,127 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:40,128 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:40,129 [254] [DEBUG] [app] Starting request: urn:request:0d6d014a-baef-441c-9a94-28765377756e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:40,130 [254] [DEBUG] [app] Ending request: urn:request:0d6d014a-baef-441c-9a94-28765377756e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0d6d014a-baef-441c-9a94-28765377756e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:09:40,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:40,130 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:40,131 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:40,131 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:40,131 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:40,135 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:40,135 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:40,139 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:40,140 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:40,142 [257] [DEBUG] [app] Ending request: urn:request:11a99bc4-490f-4476-8af8-f8e2518898bb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:11a99bc4-490f-4476-8af8-f8e2518898bb', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:40,142 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:40,142 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) repositorygcworker stdout | 2025-11-04 09:09:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:41.259858+00:00 (in 1.078686 seconds) repositorygcworker stdout | 2025-11-04 09:09:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:40 GMT)" (scheduled at 2025-11-04 09:09:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:09:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:09:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 40, 181501), True, datetime.datetime(2025, 11, 4, 9, 9, 40, 181501), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:09:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:09:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:09:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:09:40,816 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:09:41,108 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:09:41,260 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:09:41,260 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:06.265965+00:00 (in 25.005525 seconds) repositorygcworker stdout | 2025-11-04 09:09:41,260 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:41 GMT)" (scheduled at 2025-11-04 09:09:41.259858+00:00) repositorygcworker stdout | 2025-11-04 09:09:41,261 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:41,267 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), True, datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:41,268 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), True, datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 9, 41, 260753), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:09:41,270 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:09:41,270 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:41 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:09:41,407 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:42,036 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:42,042 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:42,043 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:09:42,515 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:42,915 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:43,307 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:09:44,110 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:44,551 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:44,551 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:44,553 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:44,595 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:09:44,600 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:44,679 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:44,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:47.679968+00:00 (in 2.999859 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:44,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:44 GMT)" (scheduled at 2025-11-04 09:09:44.679698+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:44,681 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:44,694 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), True, datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:44,699 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), True, datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 9, 44, 680419), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:09:44,700 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:44,701 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:14:44 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:09:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:14.980431+00:00 (in 29.999534 seconds) buildlogsarchiver stdout | 2025-11-04 09:09:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:14 GMT)" (scheduled at 2025-11-04 09:09:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:09:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 9, 44, 981154), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:09:44,987 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:09:44,987 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:09:44,987 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:09:46,035 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,114 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,146 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,149 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,164 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,168 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,169 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,171 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:09:46,177 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:49.676299+00:00 (in 1.995857 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:47 GMT)" (scheduled at 2025-11-04 09:09:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:09:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:09:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:57.766253+00:00 (in 9.999597 seconds) notificationworker stdout | 2025-11-04 09:09:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:57 GMT)" (scheduled at 2025-11-04 09:09:47.766253+00:00) notificationworker stdout | 2025-11-04 09:09:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 47, 766979), True, datetime.datetime(2025, 11, 4, 9, 9, 47, 766979), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:47,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:47,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:47,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:09:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:09:59.676299+00:00 (in 9.999498 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:59 GMT)" (scheduled at 2025-11-04 09:09:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 49, 677160), True, datetime.datetime(2025, 11, 4, 9, 9, 49, 677160), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:09:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:09:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:09:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:01.707296+00:00 (in 12.000409 seconds) securityworker stdout | 2025-11-04 09:09:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:19 GMT)" (scheduled at 2025-11-04 09:09:49.706478+00:00) securityworker stdout | 2025-11-04 09:09:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:09:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:09:49,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:09:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:09:49,715 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:49,715 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:09:49,716 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:09:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 42, 49]) securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stdout | 2025-11-04 09:09:49,720 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-32 securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:09:49,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 21, 31]) securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:09:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 2, 12]) securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:09:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Total range: 12-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 12-32 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:09:49,725 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:09:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-21 securityworker stdout | 2025-11-04 09:09:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:09:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 12, 22]) securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 31-32 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stdout | 2025-11-04 09:09:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 31, 41]) securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:09:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 38, 48]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:09:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:09:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 48, 49]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-38 securityworker stdout | 2025-11-04 09:09:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:09:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 21, 31]) securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:09:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:09:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 7, 17]) securityworker stdout | 2025-11-04 09:09:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 7-17 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Left range 7-17 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-21 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:09:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 17, 27]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 7-17 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Already merged with block 7-17 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 7-17 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 7-17 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-31 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 7-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 7-31 securityworker stdout | 2025-11-04 09:09:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stdout | 2025-11-04 09:09:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 1, 11]) securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-31 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-31 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 31-38 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-38 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-38 securityworker stdout | 2025-11-04 09:09:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 31, 41]) securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:09:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 41-31 securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:09:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 37, 47]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 37-47 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Left range 37-47 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:09:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:09:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 47, 49]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 37-47 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Already merged with block 37-47 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 37-47 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 37-47 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stdout | 2025-11-04 09:09:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 11, 21]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stdout | 2025-11-04 09:09:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:09:49,747 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:09:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-37 securityworker stdout | 2025-11-04 09:09:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:09:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 26, 36]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:09:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:09:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:09:49,749 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 26-36 securityworker stdout | 2025-11-04 09:09:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:09:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stdout | 2025-11-04 09:09:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 26-36 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stdout | 2025-11-04 09:09:49,750 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stdout | 2025-11-04 09:09:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stdout | 2025-11-04 09:09:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:09:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 1, 11]) securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 21-37 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 26-36 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Left range 26-36 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-37 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 36 securityworker stdout | 2025-11-04 09:09:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 36 securityworker stdout | 2025-11-04 09:09:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 36, 46]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:09:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 26-36 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Already merged with block 26-36 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 26 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 26-36 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 26-36 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 26 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 21-26 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-26 securityworker stdout | 2025-11-04 09:09:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-26 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-26 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 4, 49, 710107), 21, 31]) securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 31-21 securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 31 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] Total range: 31-21 securityworker stderr | 2025-11-04 09:09:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:09:49,754 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:09:49,754 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:09:50,923 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:00.086435+00:00 (in 7.925985 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:52 GMT)" (scheduled at 2025-11-04 09:09:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:09:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:09:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:09:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:33.561829+00:00 (in 40.001189 seconds) namespacegcworker stdout | 2025-11-04 09:09:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:53 GMT)" (scheduled at 2025-11-04 09:09:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:09:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:09:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 53, 561011), True, datetime.datetime(2025, 11, 4, 9, 9, 53, 561011), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:09:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:09:53,571 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:09:53,571 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:10:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:09:55,065 [254] [DEBUG] [app] Starting request: urn:request:edc7da13-5932-4989-8359-d15b23716efb (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:55,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:09:55,068 [259] [DEBUG] [app] Starting request: urn:request:6077fc70-2292-49d0-b4fc-4ffa2cb793c8 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:55,068 [259] [DEBUG] [app] Ending request: urn:request:6077fc70-2292-49d0-b4fc-4ffa2cb793c8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6077fc70-2292-49d0-b4fc-4ffa2cb793c8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:55,069 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:55,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:55,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:55,071 [261] [DEBUG] [app] Starting request: urn:request:2a06dbc7-6278-4ced-b56a-435250bd2783 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:55,071 [261] [DEBUG] [app] Ending request: urn:request:2a06dbc7-6278-4ced-b56a-435250bd2783 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2a06dbc7-6278-4ced-b56a-435250bd2783', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:55,072 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:55,072 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:55,072 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:55,072 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:55,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:09:55,076 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:55,076 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:55,081 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:55,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:55,083 [254] [DEBUG] [app] Ending request: urn:request:edc7da13-5932-4989-8359-d15b23716efb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:edc7da13-5932-4989-8359-d15b23716efb', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:55,083 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:55,083 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gunicorn-web stdout | 2025-11-04 09:09:55,122 [261] [DEBUG] [app] Starting request: urn:request:3f362d09-4337-4722-a174-d9af028879dc (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:09:55,123 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:09:55,125 [260] [DEBUG] [app] Starting request: urn:request:f61c1c21-3528-4153-868e-43021ca34801 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:09:55,125 [260] [DEBUG] [app] Ending request: urn:request:f61c1c21-3528-4153-868e-43021ca34801 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f61c1c21-3528-4153-868e-43021ca34801', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:09:55,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:09:55,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:55,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:09:55,128 [254] [DEBUG] [app] Starting request: urn:request:82c4e3f3-f17c-4db5-81e7-b256398d6ca4 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:09:55,129 [254] [DEBUG] [app] Ending request: urn:request:82c4e3f3-f17c-4db5-81e7-b256398d6ca4 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:82c4e3f3-f17c-4db5-81e7-b256398d6ca4', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:09:55,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:09:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:09:55,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:09:55,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:09:55,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:09:55,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:09:55,134 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:09:55,134 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:09:55,138 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:09:55,139 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:09:55,140 [261] [DEBUG] [app] Ending request: urn:request:3f362d09-4337-4722-a174-d9af028879dc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3f362d09-4337-4722-a174-d9af028879dc', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:09:55,140 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:09:55,140 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:09:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:09:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.019) notificationworker stdout | 2025-11-04 09:09:55,628 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:09:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:09:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:07.663502+00:00 (in 12.001426 seconds) gcworker stdout | 2025-11-04 09:09:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:25 GMT)" (scheduled at 2025-11-04 09:09:55.661677+00:00) gcworker stdout | 2025-11-04 09:09:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037795662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:09:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:09:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:09:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:09:56,023 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:09:56,205 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:09:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:09:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:09:56,587 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:09:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:09:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) notificationworker stdout | 2025-11-04 09:09:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:09:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:07.766253+00:00 (in 9.999556 seconds) notificationworker stdout | 2025-11-04 09:09:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:07 GMT)" (scheduled at 2025-11-04 09:09:57.766253+00:00) notificationworker stdout | 2025-11-04 09:09:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:09:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 57, 767066), True, datetime.datetime(2025, 11, 4, 9, 9, 57, 767066), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:09:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:09:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:09:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:09:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:09:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:09.676299+00:00 (in 9.999520 seconds) proxycacheblobworker stdout | 2025-11-04 09:09:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:09 GMT)" (scheduled at 2025-11-04 09:09:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:09:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:09:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 9, 59, 677155), True, datetime.datetime(2025, 11, 4, 9, 9, 59, 677155), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:09:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:09:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:09:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:09:59,839 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:09:59,931 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:52.159714+00:00 (in 52.072832 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:00 GMT)" (scheduled at 2025-11-04 09:10:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:10:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:00 GMT)" executed successfully builder stdout | 2025-11-04 09:10:00,220 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:10:01,030 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,170 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,170 [64] [INFO] [apscheduler.executors.default] Running job "ExpiredAppSpecificTokenWorker._gc_expired_tokens (trigger: interval[1:00:00], next run at: 2025-11-04 09:10:01 GMT)" (scheduled at 2025-11-04 09:10:01.169469+00:00) expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,170 [64] [DEBUG] [__main__] Garbage collecting expired app specific tokens with window: 1 day, 0:00:00 expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,179 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:10:01.169469+00:00 (in 3599.989615 seconds) expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,180 [64] [DEBUG] [peewee] ('DELETE FROM "appspecificauthtoken" WHERE ("appspecificauthtoken"."expiration" < %s)', [datetime.datetime(2025, 11, 3, 9, 10, 1, 171697)]) expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,185 [64] [DEBUG] [data.database] Disconnecting from database. expiredappspecifictokenworker stdout | 2025-11-04 09:10:01,185 [64] [INFO] [apscheduler.executors.default] Job "ExpiredAppSpecificTokenWorker._gc_expired_tokens (trigger: interval[1:00:00], next run at: 2025-11-04 10:10:01 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:10:01,233 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:19.706478+00:00 (in 17.998776 seconds) securityworker stdout | 2025-11-04 09:10:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:31 GMT)" (scheduled at 2025-11-04 09:10:01.707296+00:00) securityworker stdout | 2025-11-04 09:10:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:01,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:01,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:01,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:01,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 1, 712384), 1, 49]) securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 1, 712384), 1, 49]) securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:10:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:10:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:32.081579+00:00 (in 29.999559 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:10:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:32 GMT)" (scheduled at 2025-11-04 09:10:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:10:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:10:02,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:10:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:10:02,432 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:10:04,032 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:10:04,128 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:10:05,223 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:10:05,715 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:10:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:10:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:40.180734+00:00 (in 33.914194 seconds) repositorygcworker stdout | 2025-11-04 09:10:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:06 GMT)" (scheduled at 2025-11-04 09:10:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:10:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:10:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:10:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:10:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:33.568037+00:00 (in 26.998078 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:10:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:06 GMT)" (scheduled at 2025-11-04 09:10:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:10:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:10:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:10:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:10:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:37.077358+00:00 (in 29.999576 seconds) autopruneworker stdout | 2025-11-04 09:10:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:37 GMT)" (scheduled at 2025-11-04 09:10:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:10:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243807082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:10:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:10:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:10:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:10:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:25.661677+00:00 (in 17.997763 seconds) gcworker stdout | 2025-11-04 09:10:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:37 GMT)" (scheduled at 2025-11-04 09:10:07.663502+00:00) gcworker stdout | 2025-11-04 09:10:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:10:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247107670, None, 1, 0]) gcworker stdout | 2025-11-04 09:10:07,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:07,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:17.766253+00:00 (in 9.999466 seconds) notificationworker stdout | 2025-11-04 09:10:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:17 GMT)" (scheduled at 2025-11-04 09:10:07.766253+00:00) notificationworker stdout | 2025-11-04 09:10:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 7, 767113), True, datetime.datetime(2025, 11, 4, 9, 10, 7, 767113), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:10:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:10:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:07.770322+00:00 (in 59.999600 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:10:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:07 GMT)" (scheduled at 2025-11-04 09:10:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:10:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:10:07,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:07,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:07,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:10:07,775 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:10:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:10:08,233 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:10:09,330 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:09,613 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:19.676299+00:00 (in 9.999609 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:19 GMT)" (scheduled at 2025-11-04 09:10:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 9, 677067), True, datetime.datetime(2025, 11, 4, 9, 10, 9, 677067), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:19 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:10,023 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:10:10,026 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:10,065 [261] [DEBUG] [app] Starting request: urn:request:7e298b9f-5f7c-4877-aa4f-6039edce962a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:10:10,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:10:10,068 [250] [DEBUG] [app] Starting request: urn:request:02d224a2-4352-434c-b364-2d8aeb90e328 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:10,068 [250] [DEBUG] [app] Ending request: urn:request:02d224a2-4352-434c-b364-2d8aeb90e328 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:02d224a2-4352-434c-b364-2d8aeb90e328', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:10,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:10,069 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:10:10,070 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:10,071 [258] [DEBUG] [app] Starting request: urn:request:ee7cece1-1060-4478-88ca-b38575872149 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:10,072 [258] [DEBUG] [app] Ending request: urn:request:ee7cece1-1060-4478-88ca-b38575872149 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:ee7cece1-1060-4478-88ca-b38575872149', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:10:10,072 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:10,072 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:10,073 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:10,073 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:10,073 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:10,079 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:10,079 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:10,083 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:10,084 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:10,084 [261] [DEBUG] [app] Ending request: urn:request:7e298b9f-5f7c-4877-aa4f-6039edce962a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7e298b9f-5f7c-4877-aa4f-6039edce962a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:10,085 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:10,085 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gunicorn-web stdout | 2025-11-04 09:10:10,123 [261] [DEBUG] [app] Starting request: urn:request:ae58e97c-6968-4909-b34e-8c44e4d10619 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:10:10,129 [256] [DEBUG] [app] Starting request: urn:request:fa144190-e7ce-441b-97ef-655889be3573 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:10,130 [256] [DEBUG] [app] Ending request: urn:request:fa144190-e7ce-441b-97ef-655889be3573 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:fa144190-e7ce-441b-97ef-655889be3573', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.004 162 0.004) gunicorn-web stdout | 2025-11-04 09:10:10,131 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-registry stdout | 2025-11-04 09:10:10,131 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:10,132 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:10,133 [258] [DEBUG] [app] Starting request: urn:request:f8149492-6237-4ddf-b1fc-a2ad6e0665ef (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:10,133 [258] [DEBUG] [app] Ending request: urn:request:f8149492-6237-4ddf-b1fc-a2ad6e0665ef (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f8149492-6237-4ddf-b1fc-a2ad6e0665ef', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:10,133 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:10,134 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:10,134 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:10,134 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:10,134 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:10,138 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:10,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:10,142 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:10,143 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:10,144 [261] [DEBUG] [app] Ending request: urn:request:ae58e97c-6968-4909-b34e-8c44e4d10619 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ae58e97c-6968-4909-b34e-8c44e4d10619', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:10,144 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:10,144 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) quotaregistrysizeworker stdout | 2025-11-04 09:10:10,830 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:10:11,121 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:10:11,420 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:12,054 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:12,055 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:12,061 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:10:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:38.480592+00:00 (in 25.998052 seconds) exportactionlogsworker stdout | 2025-11-04 09:10:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:12 GMT)" (scheduled at 2025-11-04 09:10:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:10:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:10:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:10:12,529 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:12,927 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:13,322 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:14,124 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:14,565 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:14,572 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:14,577 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:14,610 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:14,614 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:10:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:10:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:44.980431+00:00 (in 29.999542 seconds) buildlogsarchiver stdout | 2025-11-04 09:10:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:44 GMT)" (scheduled at 2025-11-04 09:10:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:10:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 10, 14, 981198), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:10:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:10:14,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:10:16,052 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,131 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,171 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,172 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,180 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,185 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,192 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,194 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:16,194 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:27.766253+00:00 (in 9.999571 seconds) notificationworker stdout | 2025-11-04 09:10:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:27 GMT)" (scheduled at 2025-11-04 09:10:17.766253+00:00) notificationworker stdout | 2025-11-04 09:10:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 17, 767079), True, datetime.datetime(2025, 11, 4, 9, 10, 17, 767079), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:10:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:29.676299+00:00 (in 9.999573 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:29 GMT)" (scheduled at 2025-11-04 09:10:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 19, 677054), True, datetime.datetime(2025, 11, 4, 9, 10, 19, 677054), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:31.707296+00:00 (in 12.000415 seconds) securityworker stdout | 2025-11-04 09:10:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:49 GMT)" (scheduled at 2025-11-04 09:10:19.706478+00:00) securityworker stdout | 2025-11-04 09:10:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:19,714 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:10:19,714 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:10:20,935 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:25,065 [254] [DEBUG] [app] Starting request: urn:request:e99768ea-b546-45a4-b56a-8116ad582514 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:10:25,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:10:25,069 [260] [DEBUG] [app] Starting request: urn:request:3390efa1-98af-45b8-b917-fd9ebda4c4c5 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:25,069 [260] [DEBUG] [app] Ending request: urn:request:3390efa1-98af-45b8-b917-fd9ebda4c4c5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3390efa1-98af-45b8-b917-fd9ebda4c4c5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:25,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:10:25,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:25,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:25,073 [261] [DEBUG] [app] Starting request: urn:request:01641cae-7edb-4ddb-bbfb-ab2e48788103 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:25,073 [261] [DEBUG] [app] Ending request: urn:request:01641cae-7edb-4ddb-bbfb-ab2e48788103 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:01641cae-7edb-4ddb-bbfb-ab2e48788103', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:10:25,074 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:25,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:25,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:25,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:25,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:25,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:25,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:25,084 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:25,085 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:25,086 [254] [DEBUG] [app] Ending request: urn:request:e99768ea-b546-45a4-b56a-8116ad582514 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e99768ea-b546-45a4-b56a-8116ad582514', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:25,086 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:25,086 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.022) gunicorn-web stdout | 2025-11-04 09:10:25,122 [261] [DEBUG] [app] Starting request: urn:request:5e144930-b9a7-4cf9-ab18-69fe228474c3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:10:25,123 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:10:25,125 [250] [DEBUG] [app] Starting request: urn:request:6ff333cb-183f-4267-ade9-dd8c17514090 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:25,125 [250] [DEBUG] [app] Ending request: urn:request:6ff333cb-183f-4267-ade9-dd8c17514090 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6ff333cb-183f-4267-ade9-dd8c17514090', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:25,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:25,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:25,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:25,128 [254] [DEBUG] [app] Starting request: urn:request:72334c29-9315-485f-a226-30a313848413 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:25,129 [254] [DEBUG] [app] Ending request: urn:request:72334c29-9315-485f-a226-30a313848413 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:72334c29-9315-485f-a226-30a313848413', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:25,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:25,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:25,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:25,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:25,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:25,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:25,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:25,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:25,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:25,139 [261] [DEBUG] [app] Ending request: urn:request:5e144930-b9a7-4cf9-ab18-69fe228474c3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5e144930-b9a7-4cf9-ab18-69fe228474c3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:25,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:25,139 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.018) notificationworker stdout | 2025-11-04 09:10:25,640 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:37.663502+00:00 (in 12.001418 seconds) gcworker stdout | 2025-11-04 09:10:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:55 GMT)" (scheduled at 2025-11-04 09:10:25.661677+00:00) gcworker stdout | 2025-11-04 09:10:25,662 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037825662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:10:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:10:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:10:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:26,037 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:10:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:27,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:27.768179+00:00 (in 0.001007 seconds) notificationworker stdout | 2025-11-04 09:10:27,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:37 GMT)" (scheduled at 2025-11-04 09:10:27.766253+00:00) notificationworker stdout | 2025-11-04 09:10:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 27, 767475), True, datetime.datetime(2025, 11, 4, 9, 10, 27, 767475), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:37.766253+00:00 (in 9.997717 seconds) notificationworker stdout | 2025-11-04 09:10:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:27 GMT)" (scheduled at 2025-11-04 09:10:27.768179+00:00) notificationworker stdout | 2025-11-04 09:10:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:10:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:27,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:27,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:27,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:39.676299+00:00 (in 9.999535 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:39 GMT)" (scheduled at 2025-11-04 09:10:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 29, 677108), True, datetime.datetime(2025, 11, 4, 9, 10, 29, 677108), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:29,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:10:29,852 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:10:29,945 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:10:30,234 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:10:31,044 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:10:31,250 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:10:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:49.706478+00:00 (in 17.998800 seconds) securityworker stdout | 2025-11-04 09:10:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:01 GMT)" (scheduled at 2025-11-04 09:10:31.707296+00:00) securityworker stdout | 2025-11-04 09:10:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:31,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:31,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:31,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:31,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:31,717 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:31,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:31,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,720 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,720 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:31,720 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:31,720 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 31, 711759), 1, 49]) securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 31, 711759), 1, 49]) securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:10:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:10:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:10:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:10:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:02.081579+00:00 (in 29.999576 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:10:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:02 GMT)" (scheduled at 2025-11-04 09:10:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:10:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:10:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:10:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:10:32,444 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:10:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:53.560131+00:00 (in 19.997837 seconds) namespacegcworker stdout | 2025-11-04 09:10:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" (scheduled at 2025-11-04 09:10:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:10:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:10:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:10:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:10:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:06.569529+00:00 (in 33.001031 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:10:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" (scheduled at 2025-11-04 09:10:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:10:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:10:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 33, 568889), True, datetime.datetime(2025, 11, 4, 9, 10, 33, 568889), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:10:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:10:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:10:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:10:34,044 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:10:34,140 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:10:35,236 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:10:35,728 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:10:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:10:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:07.077358+00:00 (in 29.999555 seconds) autopruneworker stdout | 2025-11-04 09:10:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:07 GMT)" (scheduled at 2025-11-04 09:10:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:10:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243837083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:10:37,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:10:37,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:10:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:10:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:55.661677+00:00 (in 17.997760 seconds) gcworker stdout | 2025-11-04 09:10:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:07 GMT)" (scheduled at 2025-11-04 09:10:37.663502+00:00) gcworker stdout | 2025-11-04 09:10:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:10:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247137671, None, 1, 0]) gcworker stdout | 2025-11-04 09:10:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:37,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:47.766253+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 09:10:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:47 GMT)" (scheduled at 2025-11-04 09:10:37.766253+00:00) notificationworker stdout | 2025-11-04 09:10:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 37, 766965), True, datetime.datetime(2025, 11, 4, 9, 10, 37, 766965), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:10:38,245 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:10:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:55.481710+00:00 (in 17.000644 seconds) exportactionlogsworker stdout | 2025-11-04 09:10:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:38 GMT)" (scheduled at 2025-11-04 09:10:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:10:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:10:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 38, 481382), True, datetime.datetime(2025, 11, 4, 9, 10, 38, 481382), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:10:38,488 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:10:38,488 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:10:38,488 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:10:39,344 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:10:39,626 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:47.679968+00:00 (in 8.003210 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:49 GMT)" (scheduled at 2025-11-04 09:10:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 39, 677119), True, datetime.datetime(2025, 11, 4, 9, 10, 39, 677119), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:49 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:40,037 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:10:40,038 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:40,065 [254] [DEBUG] [app] Starting request: urn:request:94cb5bf4-9fe1-4142-ae27-df19234c7448 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:10:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:10:40,068 [250] [DEBUG] [app] Starting request: urn:request:57b25a4a-2aed-4bfa-b109-9645f9ecb23b (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:40,068 [250] [DEBUG] [app] Ending request: urn:request:57b25a4a-2aed-4bfa-b109-9645f9ecb23b (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:57b25a4a-2aed-4bfa-b109-9645f9ecb23b', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:40,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-registry stdout | 2025-11-04 09:10:40,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:40,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:40,071 [261] [DEBUG] [app] Starting request: urn:request:5e15b30f-c223-42c6-a1c9-11a99926dec3 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:40,071 [261] [DEBUG] [app] Ending request: urn:request:5e15b30f-c223-42c6-a1c9-11a99926dec3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5e15b30f-c223-42c6-a1c9-11a99926dec3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:40,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:40,072 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:40,072 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:40,072 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:40,072 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:40,076 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:40,076 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:40,080 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:40,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:40,082 [254] [DEBUG] [app] Ending request: urn:request:94cb5bf4-9fe1-4142-ae27-df19234c7448 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:94cb5bf4-9fe1-4142-ae27-df19234c7448', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:40,082 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:40,083 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.018) gunicorn-web stdout | 2025-11-04 09:10:40,122 [254] [DEBUG] [app] Starting request: urn:request:836f3ebc-c8f9-40f1-b848-17d0e07c753e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:10:40,125 [250] [DEBUG] [app] Starting request: urn:request:a929cacd-fdce-433d-afa9-2b2982d5892a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:40,125 [250] [DEBUG] [app] Ending request: urn:request:a929cacd-fdce-433d-afa9-2b2982d5892a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a929cacd-fdce-433d-afa9-2b2982d5892a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:40,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:10:40,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:40,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:40,129 [258] [DEBUG] [app] Starting request: urn:request:bfde3d90-0427-4add-aace-63728d6d02bc (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:40,129 [258] [DEBUG] [app] Ending request: urn:request:bfde3d90-0427-4add-aace-63728d6d02bc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bfde3d90-0427-4add-aace-63728d6d02bc', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:40,130 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:40,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:40,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:40,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:40,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:40,134 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:40,134 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:40,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:40,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:40,140 [254] [DEBUG] [app] Ending request: urn:request:836f3ebc-c8f9-40f1-b848-17d0e07c753e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:836f3ebc-c8f9-40f1-b848-17d0e07c753e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:40,140 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gunicorn-web stdout | 2025-11-04 09:10:40,140 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:10:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:10:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:06.265965+00:00 (in 26.084770 seconds) repositorygcworker stdout | 2025-11-04 09:10:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:40 GMT)" (scheduled at 2025-11-04 09:10:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:10:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:10:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 40, 181568), True, datetime.datetime(2025, 11, 4, 9, 10, 40, 181568), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:10:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:10:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:10:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:10:40,843 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:10:41,135 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:10:41,443 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:42,069 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:42,069 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:42,075 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:10:42,542 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:42,940 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:43,335 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:10:44,138 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:44,582 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:44,588 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:44,592 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:44,624 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:10:44,641 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:10:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:10:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:14.980431+00:00 (in 29.999533 seconds) buildlogsarchiver stdout | 2025-11-04 09:10:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:14 GMT)" (scheduled at 2025-11-04 09:10:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:10:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 10, 44, 981199), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:10:44,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:10:44,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:10:44,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:10:46,067 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,149 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,188 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,195 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,199 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,200 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,216 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,217 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:10:46,212 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:49.676299+00:00 (in 1.995833 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:47 GMT)" (scheduled at 2025-11-04 09:10:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:10:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:57.766253+00:00 (in 9.999596 seconds) notificationworker stdout | 2025-11-04 09:10:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:57 GMT)" (scheduled at 2025-11-04 09:10:47.766253+00:00) notificationworker stdout | 2025-11-04 09:10:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 47, 767033), True, datetime.datetime(2025, 11, 4, 9, 10, 47, 767033), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:47,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:47,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:47,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:10:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:10:59.676299+00:00 (in 9.999630 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:59 GMT)" (scheduled at 2025-11-04 09:10:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:49,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 49, 677058), True, datetime.datetime(2025, 11, 4, 9, 10, 49, 677058), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:49,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:49,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:49,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:10:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:10:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:10:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:01.707296+00:00 (in 12.000410 seconds) securityworker stdout | 2025-11-04 09:10:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:19 GMT)" (scheduled at 2025-11-04 09:10:49.706478+00:00) securityworker stdout | 2025-11-04 09:10:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:10:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:10:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:10:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:49,715 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:10:49,716 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,716 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,716 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:49,716 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,717 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 27, 37]) securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-27 securityworker stdout | 2025-11-04 09:10:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-27 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:10:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 15, 25]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-27 securityworker stdout | 2025-11-04 09:10:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-27 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:10:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 25, 35]) securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-37 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stdout | 2025-11-04 09:10:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stdout | 2025-11-04 09:10:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 2, 12]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:10:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:10:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 2-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:10:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:10:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:10:49,726 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 12-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-15 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-15 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:10:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 12, 22]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:10:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:10:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:10:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-37 securityworker stdout | 2025-11-04 09:10:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 37 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 39, 49]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 37-39 securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-39 securityworker stdout | 2025-11-04 09:10:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 37-39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stdout | 2025-11-04 09:10:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 37, 47]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 47-37 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:10:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 25, 35]) securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stdout | 2025-11-04 09:10:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:10:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 12, 22]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 12-22 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 12-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stdout | 2025-11-04 09:10:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stdout | 2025-11-04 09:10:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 2, 12]) securityworker stdout | 2025-11-04 09:10:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:10:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:10:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with block 12-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 12-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-22 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Left range 2-22 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-25 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-25 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:10:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 22, 32]) securityworker stdout | 2025-11-04 09:10:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:10:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:10:49,740 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-22 securityworker stdout | 2025-11-04 09:10:49,740 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-22 securityworker stdout | 2025-11-04 09:10:49,740 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-22 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-35 securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,741 [92] [DEBUG] [util.migrate.allocator] Left range 2-35 securityworker stdout | 2025-11-04 09:10:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:10:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 37, 47]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Left range 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-37 securityworker stdout | 2025-11-04 09:10:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-37 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 35 securityworker stdout | 2025-11-04 09:10:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 35, 45]) securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-35 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-47 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Right range 2-47 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 2-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:10:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 1, 11]) securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-47 securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stdout | 2025-11-04 09:10:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:10:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 2-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:10:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 47, 49]) securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 25, 35]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 25-35 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stdout | 2025-11-04 09:10:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:10:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 15, 25]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-35 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Right range 15-35 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stdout | 2025-11-04 09:10:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 25-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stdout | 2025-11-04 09:10:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 3, 13]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:10:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:10:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:10:49,754 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 15-35 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-15 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-15 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:10:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 13, 23]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:10:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-35 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Left range 3-35 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:10:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:10:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 35, 45]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:10:49,758 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-35 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-35 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-45 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-35 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Left range 3-45 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:10:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 45, 49]) securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-45 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-45 securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-45 securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 3 securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:10:49,760 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stdout | 2025-11-04 09:10:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:10:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:10:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 3 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-3 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:10:49,761 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 5, 49, 710162), 1, 11]) securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:10:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:10:49,762 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:10:49,762 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:10:50,948 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:00.086435+00:00 (in 7.925689 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:52 GMT)" (scheduled at 2025-11-04 09:10:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:10:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:10:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:10:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:33.561829+00:00 (in 40.001193 seconds) namespacegcworker stdout | 2025-11-04 09:10:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:53 GMT)" (scheduled at 2025-11-04 09:10:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:10:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:10:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 53, 560959), True, datetime.datetime(2025, 11, 4, 9, 10, 53, 560959), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:10:53,567 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:10:53,567 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:10:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:11:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:10:55,066 [254] [DEBUG] [app] Starting request: urn:request:6704cb9f-8ce5-4a72-96de-a77771938b1d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:10:55,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:10:55,069 [260] [DEBUG] [app] Starting request: urn:request:80ae22e7-6b3f-4fec-9cf9-846f18a425d2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:55,070 [260] [DEBUG] [app] Ending request: urn:request:80ae22e7-6b3f-4fec-9cf9-846f18a425d2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:80ae22e7-6b3f-4fec-9cf9-846f18a425d2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:10:55,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:55,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:55,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:55,072 [254] [DEBUG] [app] Starting request: urn:request:08b1ac33-d377-4246-9956-23bf24ec4b2c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:55,073 [254] [DEBUG] [app] Ending request: urn:request:08b1ac33-d377-4246-9956-23bf24ec4b2c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:08b1ac33-d377-4246-9956-23bf24ec4b2c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:55,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:55,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:10:55,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:55,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:55,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:55,077 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:55,077 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:55,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:55,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:55,084 [254] [DEBUG] [app] Ending request: urn:request:6704cb9f-8ce5-4a72-96de-a77771938b1d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6704cb9f-8ce5-4a72-96de-a77771938b1d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:55,084 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:55,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gunicorn-web stdout | 2025-11-04 09:10:55,122 [254] [DEBUG] [app] Starting request: urn:request:219a5cfc-cb87-4ef7-87cf-944c788b8a3e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:10:55,124 [250] [DEBUG] [app] Starting request: urn:request:e557c8c5-e90d-4bd1-9dd6-459c2d9098a7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:10:55,124 [250] [DEBUG] [app] Ending request: urn:request:e557c8c5-e90d-4bd1-9dd6-459c2d9098a7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e557c8c5-e90d-4bd1-9dd6-459c2d9098a7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:10:55,124 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:55,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:10:55,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:10:55,127 [258] [DEBUG] [app] Starting request: urn:request:91096945-e327-4609-9d52-0e39bb1d78f5 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:10:55,127 [258] [DEBUG] [app] Ending request: urn:request:91096945-e327-4609-9d52-0e39bb1d78f5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:91096945-e327-4609-9d52-0e39bb1d78f5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:10:55,128 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:10:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:10:55,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:10:55,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:10:55,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:10:55,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:10:55,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:10:55,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:10:55,137 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:10:55,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:10:55,139 [254] [DEBUG] [app] Ending request: urn:request:219a5cfc-cb87-4ef7-87cf-944c788b8a3e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:219a5cfc-cb87-4ef7-87cf-944c788b8a3e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:10:55,139 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:10:55,139 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:10:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:10:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) exportactionlogsworker stdout | 2025-11-04 09:10:55,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:10:55,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:12.482065+00:00 (in 16.999398 seconds) exportactionlogsworker stdout | 2025-11-04 09:10:55,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:15:55 GMT)" (scheduled at 2025-11-04 09:10:55.481710+00:00) exportactionlogsworker stdout | 2025-11-04 09:10:55,484 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:10:55,490 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), True, datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:10:55,503 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), True, datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 10, 55, 482968), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:10:55,504 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:10:55,505 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:15:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:10:55,653 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:10:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:10:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:07.663502+00:00 (in 12.001419 seconds) gcworker stdout | 2025-11-04 09:10:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:25 GMT)" (scheduled at 2025-11-04 09:10:55.661677+00:00) gcworker stdout | 2025-11-04 09:10:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037855662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:10:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:10:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:10:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:25 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:10:56,051 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:10:56,206 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:10:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:10:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:10:56,587 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:10:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:10:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) notificationworker stdout | 2025-11-04 09:10:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:10:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:07.766253+00:00 (in 9.999557 seconds) notificationworker stdout | 2025-11-04 09:10:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:07 GMT)" (scheduled at 2025-11-04 09:10:57.766253+00:00) notificationworker stdout | 2025-11-04 09:10:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:10:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 57, 767005), True, datetime.datetime(2025, 11, 4, 9, 10, 57, 767005), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:10:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:10:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:10:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:10:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:10:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:09.676299+00:00 (in 9.999582 seconds) proxycacheblobworker stdout | 2025-11-04 09:10:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:09 GMT)" (scheduled at 2025-11-04 09:10:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:10:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:10:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 10, 59, 677051), True, datetime.datetime(2025, 11, 4, 9, 10, 59, 677051), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:10:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:10:59,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:10:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:10:59,866 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:10:59,958 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:52.159714+00:00 (in 52.072783 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:00 GMT)" (scheduled at 2025-11-04 09:11:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:11:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:00 GMT)" executed successfully builder stdout | 2025-11-04 09:11:00,248 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:11:01,058 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:11:01,262 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:19.706478+00:00 (in 17.998751 seconds) securityworker stdout | 2025-11-04 09:11:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:31 GMT)" (scheduled at 2025-11-04 09:11:01.707296+00:00) securityworker stdout | 2025-11-04 09:11:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:01,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:01,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,720 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,720 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 1, 711505), 1, 49]) securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 1, 711505), 1, 49]) securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:01,725 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:11:01,725 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:11:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:11:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:32.081579+00:00 (in 29.999573 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:11:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:32 GMT)" (scheduled at 2025-11-04 09:11:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:11:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:11:02,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:11:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:11:02,457 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:11:04,056 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:11:04,155 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:11:05,249 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:11:05,740 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:11:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:11:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:40.180734+00:00 (in 33.914045 seconds) repositorygcworker stdout | 2025-11-04 09:11:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:06 GMT)" (scheduled at 2025-11-04 09:11:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:11:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:11:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:11:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:11:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:33.568037+00:00 (in 26.998023 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:11:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:06 GMT)" (scheduled at 2025-11-04 09:11:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:11:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:11:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:11:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:11:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:37.077358+00:00 (in 29.999573 seconds) autopruneworker stdout | 2025-11-04 09:11:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:37 GMT)" (scheduled at 2025-11-04 09:11:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:11:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243867082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:11:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:11:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:11:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:11:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:25.661677+00:00 (in 17.997747 seconds) gcworker stdout | 2025-11-04 09:11:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:37 GMT)" (scheduled at 2025-11-04 09:11:07.663502+00:00) gcworker stdout | 2025-11-04 09:11:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:11:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247167669, None, 1, 0]) gcworker stdout | 2025-11-04 09:11:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:17.766253+00:00 (in 9.999581 seconds) notificationworker stdout | 2025-11-04 09:11:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:17 GMT)" (scheduled at 2025-11-04 09:11:07.766253+00:00) notificationworker stdout | 2025-11-04 09:11:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 7, 766996), True, datetime.datetime(2025, 11, 4, 9, 11, 7, 766996), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:11:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:11:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:07.770322+00:00 (in 59.999481 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:11:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:07 GMT)" (scheduled at 2025-11-04 09:11:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:11:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:11:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:11:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:11:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:11:08,259 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:11:09,357 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:09,639 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:19.676299+00:00 (in 9.999593 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:19 GMT)" (scheduled at 2025-11-04 09:11:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 9, 677069), True, datetime.datetime(2025, 11, 4, 9, 11, 9, 677069), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:19 GMT)" executed successfully logrotateworker stdout | 2025-11-04 09:11:10,050 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:10,050 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:10,065 [258] [DEBUG] [app] Starting request: urn:request:cc047ba9-3e3d-416b-8aad-c9a0d05eb21f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:10,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:10,068 [250] [DEBUG] [app] Starting request: urn:request:8910e4ae-64cd-4142-9033-97cbc539c33f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:10,069 [250] [DEBUG] [app] Ending request: urn:request:8910e4ae-64cd-4142-9033-97cbc539c33f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8910e4ae-64cd-4142-9033-97cbc539c33f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:10,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:10,069 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:11:10,070 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:10,072 [254] [DEBUG] [app] Starting request: urn:request:32500e66-1f65-47be-829d-3bd3b42fa7aa (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:10,072 [254] [DEBUG] [app] Ending request: urn:request:32500e66-1f65-47be-829d-3bd3b42fa7aa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:32500e66-1f65-47be-829d-3bd3b42fa7aa', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:10,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:10,073 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:10,073 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:10,074 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:10,074 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:10,079 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:10,079 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:10,083 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:10,084 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:10,084 [258] [DEBUG] [app] Ending request: urn:request:cc047ba9-3e3d-416b-8aad-c9a0d05eb21f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cc047ba9-3e3d-416b-8aad-c9a0d05eb21f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:10,085 [258] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:11:10,085 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:11:10,123 [254] [DEBUG] [app] Starting request: urn:request:b8f5f6c2-c741-4ccd-83bd-74738c2a41b7 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:10,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:10,128 [250] [DEBUG] [app] Starting request: urn:request:94c7c526-d8d8-4ba9-b753-b323db488664 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:10,128 [250] [DEBUG] [app] Ending request: urn:request:94c7c526-d8d8-4ba9-b753-b323db488664 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:94c7c526-d8d8-4ba9-b753-b323db488664', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:11:10,129 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:10,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:10,130 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:10,131 [258] [DEBUG] [app] Starting request: urn:request:b4212379-7e06-413a-8b24-cec03d08d72b (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:10,131 [258] [DEBUG] [app] Ending request: urn:request:b4212379-7e06-413a-8b24-cec03d08d72b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b4212379-7e06-413a-8b24-cec03d08d72b', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:10,132 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:10,132 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:10,132 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:10,132 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:10,132 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:10,136 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:10,136 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:10,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:10,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:10,142 [254] [DEBUG] [app] Ending request: urn:request:b8f5f6c2-c741-4ccd-83bd-74738c2a41b7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b8f5f6c2-c741-4ccd-83bd-74738c2a41b7', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:10,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:10,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.020) quotaregistrysizeworker stdout | 2025-11-04 09:11:10,857 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:11:11,148 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:11:11,455 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:12,084 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:12,084 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:12,089 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:11:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:38.480592+00:00 (in 25.998080 seconds) exportactionlogsworker stdout | 2025-11-04 09:11:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:12 GMT)" (scheduled at 2025-11-04 09:11:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:11:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:11:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:11:12,555 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:12,953 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:11:13,360 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:14,152 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:14,600 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:14,610 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:14,602 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:14,638 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:14,655 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:11:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:11:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:44.980431+00:00 (in 29.999568 seconds) buildlogsarchiver stdout | 2025-11-04 09:11:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:44 GMT)" (scheduled at 2025-11-04 09:11:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:11:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 11, 14, 981143), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:11:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:11:14,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:11:16,083 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,166 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,206 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,222 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,227 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,233 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,218 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,241 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:16,233 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:27.766253+00:00 (in 9.999585 seconds) notificationworker stdout | 2025-11-04 09:11:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:27 GMT)" (scheduled at 2025-11-04 09:11:17.766253+00:00) notificationworker stdout | 2025-11-04 09:11:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 17, 767040), True, datetime.datetime(2025, 11, 4, 9, 11, 17, 767040), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:17,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:17,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:11:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:29.676299+00:00 (in 9.999574 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:29 GMT)" (scheduled at 2025-11-04 09:11:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 19, 677049), True, datetime.datetime(2025, 11, 4, 9, 11, 19, 677049), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:11:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:31.707296+00:00 (in 12.000394 seconds) securityworker stdout | 2025-11-04 09:11:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:49 GMT)" (scheduled at 2025-11-04 09:11:19.706478+00:00) securityworker stdout | 2025-11-04 09:11:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:19,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:11:19,716 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:11:20,961 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:25,066 [258] [DEBUG] [app] Starting request: urn:request:b5d3d858-2f8c-4d8b-a4f7-f208b833bbeb (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:25,073 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:25,076 [250] [DEBUG] [app] Starting request: urn:request:48795b3b-ec5c-4954-aff6-121ef34ad96d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:25,076 [250] [DEBUG] [app] Ending request: urn:request:48795b3b-ec5c-4954-aff6-121ef34ad96d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:48795b3b-ec5c-4954-aff6-121ef34ad96d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:25,076 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:11:25,077 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:25,078 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:25,079 [258] [DEBUG] [app] Starting request: urn:request:4101d717-d92d-420f-8c9b-4b5c43e7cf91 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:25,080 [258] [DEBUG] [app] Ending request: urn:request:4101d717-d92d-420f-8c9b-4b5c43e7cf91 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4101d717-d92d-420f-8c9b-4b5c43e7cf91', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:25,080 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:11:25,081 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:25,081 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:25,081 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:25,081 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:25,086 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:25,086 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:25,090 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:25,092 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:25,092 [258] [DEBUG] [app] Ending request: urn:request:b5d3d858-2f8c-4d8b-a4f7-f208b833bbeb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b5d3d858-2f8c-4d8b-a4f7-f208b833bbeb', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:25,092 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:25,093 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.028 118 0.028) gunicorn-web stdout | 2025-11-04 09:11:25,123 [257] [DEBUG] [app] Starting request: urn:request:fc051f78-1d10-4aae-9306-7f6d2365db42 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:25,124 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:25,126 [260] [DEBUG] [app] Starting request: urn:request:ccbbf03a-5ca0-4bd2-a6f0-352918193636 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:25,126 [260] [DEBUG] [app] Ending request: urn:request:ccbbf03a-5ca0-4bd2-a6f0-352918193636 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ccbbf03a-5ca0-4bd2-a6f0-352918193636', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:11:25,127 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:25,127 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:25,128 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:25,129 [254] [DEBUG] [app] Starting request: urn:request:c8e5d4bc-fbbc-4c6e-8b07-35e084c409fa (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:25,130 [254] [DEBUG] [app] Ending request: urn:request:c8e5d4bc-fbbc-4c6e-8b07-35e084c409fa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c8e5d4bc-fbbc-4c6e-8b07-35e084c409fa', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:25,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:11:25,130 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:25,131 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:25,131 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:25,131 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:25,135 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:25,135 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:25,140 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:25,142 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:25,142 [257] [DEBUG] [app] Ending request: urn:request:fc051f78-1d10-4aae-9306-7f6d2365db42 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fc051f78-1d10-4aae-9306-7f6d2365db42', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:25,143 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:25,143 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gcworker stdout | 2025-11-04 09:11:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:37.663502+00:00 (in 12.001408 seconds) gcworker stdout | 2025-11-04 09:11:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:55 GMT)" (scheduled at 2025-11-04 09:11:25.661677+00:00) gcworker stdout | 2025-11-04 09:11:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037885662, 1209600, True, 3, 500, 1, 0]) notificationworker stdout | 2025-11-04 09:11:25,666 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:11:25,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:11:25,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:25,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:11:55 GMT)" executed successfully buildlogsarchiver stdout | 2025-11-04 09:11:26,065 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:11:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:27.768179+00:00 (in 0.001506 seconds) notificationworker stdout | 2025-11-04 09:11:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:37 GMT)" (scheduled at 2025-11-04 09:11:27.766253+00:00) notificationworker stdout | 2025-11-04 09:11:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 27, 767009), True, datetime.datetime(2025, 11, 4, 9, 11, 27, 767009), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:37.766253+00:00 (in 9.997716 seconds) notificationworker stdout | 2025-11-04 09:11:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:27 GMT)" (scheduled at 2025-11-04 09:11:27.768179+00:00) notificationworker stdout | 2025-11-04 09:11:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:11:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:39.676299+00:00 (in 9.999580 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:39 GMT)" (scheduled at 2025-11-04 09:11:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 29, 677045), True, datetime.datetime(2025, 11, 4, 9, 11, 29, 677045), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:11:29,879 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:11:29,973 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:11:30,261 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:11:31,071 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:11:31,274 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:11:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:49.706478+00:00 (in 17.998766 seconds) securityworker stdout | 2025-11-04 09:11:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:01 GMT)" (scheduled at 2025-11-04 09:11:31.707296+00:00) securityworker stdout | 2025-11-04 09:11:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:31,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 31, 712770), 1, 49]) securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 31, 712770), 1, 49]) securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:11:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:11:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:11:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:11:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:02.081579+00:00 (in 29.999580 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:11:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:02 GMT)" (scheduled at 2025-11-04 09:11:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:11:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:11:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:11:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:11:32,469 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:11:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:53.560131+00:00 (in 19.997867 seconds) namespacegcworker stdout | 2025-11-04 09:11:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" (scheduled at 2025-11-04 09:11:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:11:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:11:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:11:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:11:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:06.569529+00:00 (in 33.001052 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:11:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" (scheduled at 2025-11-04 09:11:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:11:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:11:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 33, 568853), True, datetime.datetime(2025, 11, 4, 9, 11, 33, 568853), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:11:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:11:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:11:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:11:34,070 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:11:34,168 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:11:35,262 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:11:35,754 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:11:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:11:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:07.077358+00:00 (in 29.999523 seconds) autopruneworker stdout | 2025-11-04 09:11:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:07 GMT)" (scheduled at 2025-11-04 09:11:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:11:37,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243897082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:11:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:11:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:11:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:11:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:55.661677+00:00 (in 17.997734 seconds) gcworker stdout | 2025-11-04 09:11:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:07 GMT)" (scheduled at 2025-11-04 09:11:37.663502+00:00) gcworker stdout | 2025-11-04 09:11:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:11:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247197670, None, 1, 0]) gcworker stdout | 2025-11-04 09:11:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:37,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:47.766253+00:00 (in 9.999579 seconds) notificationworker stdout | 2025-11-04 09:11:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:47 GMT)" (scheduled at 2025-11-04 09:11:37.766253+00:00) notificationworker stdout | 2025-11-04 09:11:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 37, 767012), True, datetime.datetime(2025, 11, 4, 9, 11, 37, 767012), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:11:38,271 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:11:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:12.482065+00:00 (in 34.000972 seconds) exportactionlogsworker stdout | 2025-11-04 09:11:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:38 GMT)" (scheduled at 2025-11-04 09:11:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:11:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:11:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 38, 481543), True, datetime.datetime(2025, 11, 4, 9, 11, 38, 481543), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:11:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:11:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:11:38,489 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:11:39,370 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:11:39,651 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:47.679968+00:00 (in 8.003290 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:49 GMT)" (scheduled at 2025-11-04 09:11:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 39, 677045), True, datetime.datetime(2025, 11, 4, 9, 11, 39, 677045), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:49 GMT)" executed successfully securityworker stdout | 2025-11-04 09:11:40,064 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:40,065 [258] [DEBUG] [app] Starting request: urn:request:8d15aad0-0faf-4579-b0cc-d532d2f4bc3f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} logrotateworker stdout | 2025-11-04 09:11:40,065 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:40,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:40,069 [260] [DEBUG] [app] Starting request: urn:request:dce00749-e5d4-472d-b826-156263c50953 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:40,070 [260] [DEBUG] [app] Ending request: urn:request:dce00749-e5d4-472d-b826-156263c50953 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:dce00749-e5d4-472d-b826-156263c50953', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:40,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:40,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:11:40,071 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:40,072 [261] [DEBUG] [app] Starting request: urn:request:1320dd5f-92e8-4d2f-9d36-86931f47f75e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:40,073 [261] [DEBUG] [app] Ending request: urn:request:1320dd5f-92e8-4d2f-9d36-86931f47f75e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1320dd5f-92e8-4d2f-9d36-86931f47f75e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:40,073 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:40,073 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:11:40,074 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:40,074 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:40,074 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:40,078 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:40,078 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:40,082 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:40,083 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:40,084 [258] [DEBUG] [app] Ending request: urn:request:8d15aad0-0faf-4579-b0cc-d532d2f4bc3f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8d15aad0-0faf-4579-b0cc-d532d2f4bc3f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:40,084 [258] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.020) gunicorn-web stdout | 2025-11-04 09:11:40,085 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:11:40,123 [254] [DEBUG] [app] Starting request: urn:request:5968e9ab-a954-4a20-8da9-b97c84038586 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:40,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:40,126 [260] [DEBUG] [app] Starting request: urn:request:f6c5c448-b92d-4317-a10e-17326a1d938e (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:40,126 [260] [DEBUG] [app] Ending request: urn:request:f6c5c448-b92d-4317-a10e-17326a1d938e (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f6c5c448-b92d-4317-a10e-17326a1d938e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:40,127 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:40,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:11:40,128 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:40,130 [258] [DEBUG] [app] Starting request: urn:request:3325beb3-891f-4dbd-97d3-a97202518cf9 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:40,130 [258] [DEBUG] [app] Ending request: urn:request:3325beb3-891f-4dbd-97d3-a97202518cf9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3325beb3-891f-4dbd-97d3-a97202518cf9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:40,130 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:40,131 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:40,131 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:40,131 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:40,131 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:40,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:40,136 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:40,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:40,142 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:40,143 [254] [DEBUG] [app] Ending request: urn:request:5968e9ab-a954-4a20-8da9-b97c84038586 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5968e9ab-a954-4a20-8da9-b97c84038586', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:40,143 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:40,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) repositorygcworker stdout | 2025-11-04 09:11:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:11:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:06.265965+00:00 (in 26.084812 seconds) repositorygcworker stdout | 2025-11-04 09:11:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:40 GMT)" (scheduled at 2025-11-04 09:11:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:11:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:11:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 40, 181509), True, datetime.datetime(2025, 11, 4, 9, 11, 40, 181509), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:11:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:11:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:11:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:11:40,883 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:11:41,162 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:11:41,468 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:42,103 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:42,107 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:42,107 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:11:42,568 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:42,965 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:11:43,373 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:11:44,167 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:44,615 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:44,625 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:44,629 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:44,653 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:11:44,668 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:11:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:11:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:14.980431+00:00 (in 29.999587 seconds) buildlogsarchiver stdout | 2025-11-04 09:11:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:14 GMT)" (scheduled at 2025-11-04 09:11:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:11:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 11, 44, 981119), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:11:44,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:11:44,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:11:44,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:11:46,098 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,180 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,242 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,244 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,253 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,254 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,255 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,259 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:11:46,264 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:49.676299+00:00 (in 1.995839 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:47 GMT)" (scheduled at 2025-11-04 09:11:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:11:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:47,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:57.766253+00:00 (in 9.999179 seconds) notificationworker stdout | 2025-11-04 09:11:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:57 GMT)" (scheduled at 2025-11-04 09:11:47.766253+00:00) notificationworker stdout | 2025-11-04 09:11:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 47, 767392), True, datetime.datetime(2025, 11, 4, 9, 11, 47, 767392), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:47,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:47,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:47,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:11:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:11:59.676299+00:00 (in 9.999503 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:59 GMT)" (scheduled at 2025-11-04 09:11:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 49, 677145), True, datetime.datetime(2025, 11, 4, 9, 11, 49, 677145), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:11:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:11:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:11:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:01.707296+00:00 (in 12.000411 seconds) securityworker stdout | 2025-11-04 09:11:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:19 GMT)" (scheduled at 2025-11-04 09:11:49.706478+00:00) securityworker stdout | 2025-11-04 09:11:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:11:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:11:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:11:49,709 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:49,714 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:11:49,714 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,714 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,714 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:49,714 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,715 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:11:49,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 42, 49]) securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stdout | 2025-11-04 09:11:49,718 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-32 securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:11:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 4, 14]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:11:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:49,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 14-32 securityworker stdout | 2025-11-04 09:11:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-32 securityworker stdout | 2025-11-04 09:11:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 14-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:11:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 20, 30]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 14-32 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 14-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-20 securityworker stdout | 2025-11-04 09:11:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:11:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 14, 24]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Merging with block 20-30 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 30 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 30-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 20-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 30-32 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-32 securityworker stdout | 2025-11-04 09:11:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stdout | 2025-11-04 09:11:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 30, 40]) securityworker stdout | 2025-11-04 09:11:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stdout | 2025-11-04 09:11:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 30 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 40 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 40-30 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 40 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 40-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 21, 31]) securityworker stdout | 2025-11-04 09:11:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 32, 42]) securityworker stdout | 2025-11-04 09:11:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:11:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:11:49,731 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:11:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:11:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 31, 41]) securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-31 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-31 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-42 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Right range 21-42 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:11:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-31 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:11:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 10, 20]) securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stdout | 2025-11-04 09:11:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:11:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 1, 11]) securityworker stdout | 2025-11-04 09:11:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:11:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:11:49,736 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 20-49 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 20-49 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 21-42 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-21 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-21 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:11:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 20, 30]) securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:11:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:11:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 42, 49]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:11:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 18, 28]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:11:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Left range 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Left range 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stdout | 2025-11-04 09:11:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:11:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 35, 45]) securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stdout | 2025-11-04 09:11:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stdout | 2025-11-04 09:11:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 1, 11]) securityworker stdout | 2025-11-04 09:11:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:11:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-18 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-18 securityworker stdout | 2025-11-04 09:11:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:11:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 11, 21]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Right range 35-45 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-35 securityworker stdout | 2025-11-04 09:11:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 28-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Right range 35-45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-35 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:11:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 28, 38]) securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Merging with block 35-45 securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 45 securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 45-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 35-45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 45-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:11:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:11:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 6, 49, 709737), 45, 49]) securityworker stdout | 2025-11-04 09:11:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:11:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:11:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:11:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:11:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 49-45 securityworker stdout | 2025-11-04 09:11:49,753 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:49,753 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-45 securityworker stderr | 2025-11-04 09:11:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:11:49,753 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:11:50,974 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:00.086435+00:00 (in 7.925940 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:52 GMT)" (scheduled at 2025-11-04 09:11:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:11:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:11:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:11:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:32.561493+00:00 (in 39.000862 seconds) namespacegcworker stdout | 2025-11-04 09:11:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:53 GMT)" (scheduled at 2025-11-04 09:11:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:11:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:11:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 53, 561012), True, datetime.datetime(2025, 11, 4, 9, 11, 53, 561012), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:11:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:11:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:11:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:12:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:11:55,065 [254] [DEBUG] [app] Starting request: urn:request:bd5f6a47-6653-4390-839e-831b97526bd8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:55,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:55,068 [260] [DEBUG] [app] Starting request: urn:request:5e34e6c8-2176-467b-ad7e-c74fa6c44fac (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:55,068 [260] [DEBUG] [app] Ending request: urn:request:5e34e6c8-2176-467b-ad7e-c74fa6c44fac (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5e34e6c8-2176-467b-ad7e-c74fa6c44fac', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:11:55,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:11:55,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:55,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:55,076 [254] [DEBUG] [app] Starting request: urn:request:7af63aa0-c16a-4bbb-8dd7-f12caf77295f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:55,076 [254] [DEBUG] [app] Ending request: urn:request:7af63aa0-c16a-4bbb-8dd7-f12caf77295f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:7af63aa0-c16a-4bbb-8dd7-f12caf77295f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:11:55,077 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:55,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:55,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:11:55,077 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:55,077 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:55,083 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:55,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:55,087 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:55,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:55,088 [254] [DEBUG] [app] Ending request: urn:request:bd5f6a47-6653-4390-839e-831b97526bd8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bd5f6a47-6653-4390-839e-831b97526bd8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:55,089 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:55,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.025 118 0.024) gunicorn-web stdout | 2025-11-04 09:11:55,122 [257] [DEBUG] [app] Starting request: urn:request:d798562e-f790-4374-a9d3-1b2fe7f2747f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:11:55,123 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:11:55,126 [250] [DEBUG] [app] Starting request: urn:request:11598326-fc16-414b-a3f1-efa167706259 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:11:55,126 [250] [DEBUG] [app] Ending request: urn:request:11598326-fc16-414b-a3f1-efa167706259 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:11598326-fc16-414b-a3f1-efa167706259', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:11:55,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:55,127 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:55,128 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:11:55,129 [254] [DEBUG] [app] Starting request: urn:request:059bf921-d9fb-451e-8fca-71fb6805a3e1 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:11:55,130 [254] [DEBUG] [app] Ending request: urn:request:059bf921-d9fb-451e-8fca-71fb6805a3e1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:059bf921-d9fb-451e-8fca-71fb6805a3e1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:11:55,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:11:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:11:55,130 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:11:55,130 [257] [DEBUG] [data.model.health] Validating database connection. nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:11:55,131 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:11:55,131 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:11:55,135 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:11:55,135 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:11:55,139 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:11:55,140 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:11:55,140 [257] [DEBUG] [app] Ending request: urn:request:d798562e-f790-4374-a9d3-1b2fe7f2747f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d798562e-f790-4374-a9d3-1b2fe7f2747f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:11:55,141 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:11:55,141 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:11:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:11:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gcworker stdout | 2025-11-04 09:11:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:11:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:07.663502+00:00 (in 12.001393 seconds) gcworker stdout | 2025-11-04 09:11:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:25 GMT)" (scheduled at 2025-11-04 09:11:55.661677+00:00) gcworker stdout | 2025-11-04 09:11:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037915662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:11:55,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:11:55,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:11:55,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:11:55,682 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:11:56,091 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:11:56,205 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:11:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:11:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:11:56,591 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:11:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:11:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.000) notificationworker stdout | 2025-11-04 09:11:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:11:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:07.766253+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 09:11:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:07 GMT)" (scheduled at 2025-11-04 09:11:57.766253+00:00) notificationworker stdout | 2025-11-04 09:11:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:11:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 57, 767054), True, datetime.datetime(2025, 11, 4, 9, 11, 57, 767054), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:11:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:11:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:11:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:11:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:11:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:09.676299+00:00 (in 9.999519 seconds) proxycacheblobworker stdout | 2025-11-04 09:11:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:09 GMT)" (scheduled at 2025-11-04 09:11:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:11:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:11:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 11, 59, 677113), True, datetime.datetime(2025, 11, 4, 9, 11, 59, 677113), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:11:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:11:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:11:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:11:59,891 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:11:59,988 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:52.159714+00:00 (in 52.072785 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:00 GMT)" (scheduled at 2025-11-04 09:12:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,095 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:12:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:00 GMT)" executed successfully builder stdout | 2025-11-04 09:12:00,275 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:12:01,085 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:12:01,286 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:19.706478+00:00 (in 17.998743 seconds) securityworker stdout | 2025-11-04 09:12:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:31 GMT)" (scheduled at 2025-11-04 09:12:01.707296+00:00) securityworker stdout | 2025-11-04 09:12:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:01,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 1, 713114), 1, 49]) securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 1, 713114), 1, 49]) securityworker stdout | 2025-11-04 09:12:01,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:01,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:12:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:12:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:32.081579+00:00 (in 29.999577 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:12:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:32 GMT)" (scheduled at 2025-11-04 09:12:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:12:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:12:02,083 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:12:02,084 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:12:02,481 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:12:04,084 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:12:04,181 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:12:05,274 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:12:05,768 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:12:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:12:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:40.180734+00:00 (in 33.914220 seconds) repositorygcworker stdout | 2025-11-04 09:12:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:06 GMT)" (scheduled at 2025-11-04 09:12:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:12:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:12:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:12:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:28.569177+00:00 (in 21.999210 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:06 GMT)" (scheduled at 2025-11-04 09:12:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:12:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:12:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:12:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:37.077358+00:00 (in 29.999554 seconds) autopruneworker stdout | 2025-11-04 09:12:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:37 GMT)" (scheduled at 2025-11-04 09:12:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:12:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243927082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:12:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:12:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:12:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:12:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:25.661677+00:00 (in 17.997741 seconds) gcworker stdout | 2025-11-04 09:12:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:37 GMT)" (scheduled at 2025-11-04 09:12:07.663502+00:00) gcworker stdout | 2025-11-04 09:12:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:12:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247227670, None, 1, 0]) gcworker stdout | 2025-11-04 09:12:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:07,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:17.766253+00:00 (in 9.999585 seconds) notificationworker stdout | 2025-11-04 09:12:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:17 GMT)" (scheduled at 2025-11-04 09:12:07.766253+00:00) notificationworker stdout | 2025-11-04 09:12:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 7, 767057), True, datetime.datetime(2025, 11, 4, 9, 12, 7, 767057), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:12:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:12:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:07.770322+00:00 (in 59.999519 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:12:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:07 GMT)" (scheduled at 2025-11-04 09:12:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:12:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:12:07,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:12:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:12:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:12:08,284 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:12:09,384 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:12:09,663 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:19.676299+00:00 (in 9.999587 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:19 GMT)" (scheduled at 2025-11-04 09:12:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 9, 677033), True, datetime.datetime(2025, 11, 4, 9, 12, 9, 677033), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:19 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:10,065 [258] [DEBUG] [app] Starting request: urn:request:9ad1f152-23ac-474c-835c-b59b90f2c71f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:12:10,066 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:12:10,068 [260] [DEBUG] [app] Starting request: urn:request:90df127e-66cb-4fcc-aa43-a32b50227e19 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:10,068 [260] [DEBUG] [app] Ending request: urn:request:90df127e-66cb-4fcc-aa43-a32b50227e19 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:90df127e-66cb-4fcc-aa43-a32b50227e19', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:10,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:10,069 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:10,070 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:10,075 [258] [DEBUG] [app] Starting request: urn:request:3c707f07-2527-430a-ace2-05cd0d3a4f8c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:10,076 [258] [DEBUG] [app] Ending request: urn:request:3c707f07-2527-430a-ace2-05cd0d3a4f8c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3c707f07-2527-430a-ace2-05cd0d3a4f8c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.005 159 0.005) gunicorn-web stdout | 2025-11-04 09:12:10,076 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:10,077 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:10,077 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:10,077 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:10,077 [258] [INFO] [data.database] Connection pooling disabled for postgresql securityworker stdout | 2025-11-04 09:12:10,077 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:10,082 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:10,082 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) logrotateworker stdout | 2025-11-04 09:12:10,082 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:10,086 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:10,087 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:10,088 [258] [DEBUG] [app] Ending request: urn:request:9ad1f152-23ac-474c-835c-b59b90f2c71f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9ad1f152-23ac-474c-835c-b59b90f2c71f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:10,088 [258] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:12:10,088 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:12:10,122 [258] [DEBUG] [app] Starting request: urn:request:adfcfaa2-1e94-4ae1-9181-921e42f05604 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:12:10,124 [260] [DEBUG] [app] Starting request: urn:request:d3f093da-5cd1-4865-982e-3488d9725b20 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:10,124 [260] [DEBUG] [app] Ending request: urn:request:d3f093da-5cd1-4865-982e-3488d9725b20 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d3f093da-5cd1-4865-982e-3488d9725b20', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:10,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:10,125 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:10,126 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:10,128 [254] [DEBUG] [app] Starting request: urn:request:9ea7b568-8259-4e84-ac5c-29c88e712c39 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:10,128 [254] [DEBUG] [app] Ending request: urn:request:9ea7b568-8259-4e84-ac5c-29c88e712c39 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9ea7b568-8259-4e84-ac5c-29c88e712c39', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:10,128 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:10,129 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:10,129 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:10,129 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:10,129 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:10,133 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:10,133 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:10,137 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:10,138 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:10,139 [258] [DEBUG] [app] Ending request: urn:request:adfcfaa2-1e94-4ae1-9181-921e42f05604 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:adfcfaa2-1e94-4ae1-9181-921e42f05604', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:10,139 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:10,139 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.018) quotaregistrysizeworker stdout | 2025-11-04 09:12:10,895 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:12:11,175 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:12:11,480 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:12,117 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:12,122 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:12,122 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:12:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:12:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:38.480592+00:00 (in 25.998057 seconds) exportactionlogsworker stdout | 2025-11-04 09:12:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:12 GMT)" (scheduled at 2025-11-04 09:12:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:12:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:12:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:12:12,582 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:12,978 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:12:13,385 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:14,180 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:14,632 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:14,640 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:14,643 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:14,668 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:14,682 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:12:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:12:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:44.980431+00:00 (in 29.999583 seconds) buildlogsarchiver stdout | 2025-11-04 09:12:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:44 GMT)" (scheduled at 2025-11-04 09:12:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:12:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 12, 14, 981167), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:12:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:12:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:12:16,123 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,195 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,258 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,263 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,268 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,273 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,277 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,279 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:16,281 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:19.767846+00:00 (in 2.001164 seconds) notificationworker stdout | 2025-11-04 09:12:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:27 GMT)" (scheduled at 2025-11-04 09:12:17.766253+00:00) notificationworker stdout | 2025-11-04 09:12:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 17, 766963), True, datetime.datetime(2025, 11, 4, 9, 12, 17, 766963), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:17,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:17,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:17,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:12:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:29.676299+00:00 (in 9.999575 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:29 GMT)" (scheduled at 2025-11-04 09:12:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 19, 677380), True, datetime.datetime(2025, 11, 4, 9, 12, 19, 677380), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:12:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:31.707296+00:00 (in 12.000416 seconds) securityworker stdout | 2025-11-04 09:12:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:49 GMT)" (scheduled at 2025-11-04 09:12:19.706478+00:00) securityworker stdout | 2025-11-04 09:12:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:19,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:12:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:12:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:49 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:19,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:19,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:27.766253+00:00 (in 7.997987 seconds) notificationworker stdout | 2025-11-04 09:12:19,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:19 GMT)" (scheduled at 2025-11-04 09:12:19.767846+00:00) notificationworker stdout | 2025-11-04 09:12:19,769 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), 'notification/%']) notificationworker stdout | 2025-11-04 09:12:19,775 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), True, datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:12:19,776 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), True, datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 12, 19, 768634), 'notification/%']) notificationworker stdout | 2025-11-04 09:12:19,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:19,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:12:20,988 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:25,066 [254] [DEBUG] [app] Starting request: urn:request:29fe0fdf-7fb4-497d-9463-03c4f9ec37f5 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:12:25,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:12:25,069 [256] [DEBUG] [app] Starting request: urn:request:b5e25c19-902a-4e51-9d66-cdad35e66afe (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:25,069 [256] [DEBUG] [app] Ending request: urn:request:b5e25c19-902a-4e51-9d66-cdad35e66afe (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b5e25c19-902a-4e51-9d66-cdad35e66afe', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:25,070 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:25,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:25,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:25,073 [254] [DEBUG] [app] Starting request: urn:request:dfb9882d-b4e4-4129-aee7-f150834a33b6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:25,073 [254] [DEBUG] [app] Ending request: urn:request:dfb9882d-b4e4-4129-aee7-f150834a33b6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dfb9882d-b4e4-4129-aee7-f150834a33b6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:25,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:25,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:25,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:25,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:25,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:25,078 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:25,078 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:25,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:25,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:25,084 [254] [DEBUG] [app] Ending request: urn:request:29fe0fdf-7fb4-497d-9463-03c4f9ec37f5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:29fe0fdf-7fb4-497d-9463-03c4f9ec37f5', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:25,084 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.020) gunicorn-web stdout | 2025-11-04 09:12:25,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:12:25,122 [254] [DEBUG] [app] Starting request: urn:request:16a7c498-abc8-45cd-abf7-8ef2640d2ce9 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:12:25,123 [250] [DEBUG] [app] Starting request: urn:request:f7aa5894-f1dd-4625-a5e2-36f88796130d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:25,124 [250] [DEBUG] [app] Ending request: urn:request:f7aa5894-f1dd-4625-a5e2-36f88796130d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f7aa5894-f1dd-4625-a5e2-36f88796130d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:25,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:25,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:12:25,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:25,127 [258] [DEBUG] [app] Starting request: urn:request:2e24f094-403e-4b07-971d-1011095281ff (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:25,127 [258] [DEBUG] [app] Ending request: urn:request:2e24f094-403e-4b07-971d-1011095281ff (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2e24f094-403e-4b07-971d-1011095281ff', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:25,128 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:25,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:25,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:25,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:25,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:25,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:25,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:25,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:25,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:25,138 [254] [DEBUG] [app] Ending request: urn:request:16a7c498-abc8-45cd-abf7-8ef2640d2ce9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:16a7c498-abc8-45cd-abf7-8ef2640d2ce9', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:25,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:25,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.017) gcworker stdout | 2025-11-04 09:12:25,662 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:37.663502+00:00 (in 12.001134 seconds) gcworker stdout | 2025-11-04 09:12:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:55 GMT)" (scheduled at 2025-11-04 09:12:25.661677+00:00) gcworker stdout | 2025-11-04 09:12:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037945662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:12:25,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:12:25,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:25,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:12:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:25,696 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:12:26,103 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:12:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:27.768179+00:00 (in 0.001474 seconds) notificationworker stdout | 2025-11-04 09:12:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:37 GMT)" (scheduled at 2025-11-04 09:12:27.766253+00:00) notificationworker stdout | 2025-11-04 09:12:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 27, 767058), True, datetime.datetime(2025, 11, 4, 9, 12, 27, 767058), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:37.766253+00:00 (in 9.997433 seconds) notificationworker stdout | 2025-11-04 09:12:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:27 GMT)" (scheduled at 2025-11-04 09:12:27.768179+00:00) notificationworker stdout | 2025-11-04 09:12:27,769 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:12:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:12:28,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:28,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:33.568037+00:00 (in 4.998378 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:28,569 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:28 GMT)" (scheduled at 2025-11-04 09:12:28.569177+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:28,570 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:28,576 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), True, datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:28,577 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), True, datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 12, 28, 569934), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:12:28,579 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:12:28,579 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:28 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:12:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:39.676299+00:00 (in 9.999570 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:39 GMT)" (scheduled at 2025-11-04 09:12:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 29, 676999), True, datetime.datetime(2025, 11, 4, 9, 12, 29, 676999), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:29,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:12:29,905 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:12:30,015 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:12:30,288 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:12:31,099 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:12:31,298 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:12:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:49.706478+00:00 (in 17.998777 seconds) securityworker stdout | 2025-11-04 09:12:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:01 GMT)" (scheduled at 2025-11-04 09:12:31.707296+00:00) securityworker stdout | 2025-11-04 09:12:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:31,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:31,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:31,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:31,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:31,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:31,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:31,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:31,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:31,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:12:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 31, 714460), 1, 49]) securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:31,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:31,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:31,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:31,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 31, 714460), 1, 49]) securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:31,729 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:12:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:31,729 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:12:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:12:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:02.081579+00:00 (in 29.999583 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:12:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:02 GMT)" (scheduled at 2025-11-04 09:12:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:12:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:12:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:12:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:12:32,493 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:32,561 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:12:32,561 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:33.561829+00:00 (in 0.999873 seconds) namespacegcworker stdout | 2025-11-04 09:12:32,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:32 GMT)" (scheduled at 2025-11-04 09:12:32.561493+00:00) namespacegcworker stdout | 2025-11-04 09:12:32,562 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:12:32,569 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), True, datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), 0, 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:12:32,570 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), True, datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 12, 32, 562246), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:12:32,571 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:12:32,571 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:17:32 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:12:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:12:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:53.560131+00:00 (in 19.997885 seconds) namespacegcworker stdout | 2025-11-04 09:12:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" (scheduled at 2025-11-04 09:12:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:12:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:12:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:12:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:12:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:06.569529+00:00 (in 33.000988 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:12:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" (scheduled at 2025-11-04 09:12:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:12:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:12:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 33, 568936), True, datetime.datetime(2025, 11, 4, 9, 12, 33, 568936), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:12:33,575 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:12:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:12:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:12:34,097 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:12:34,195 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:12:35,286 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:12:35,782 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:12:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:12:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:07.077358+00:00 (in 29.999566 seconds) autopruneworker stdout | 2025-11-04 09:12:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:07 GMT)" (scheduled at 2025-11-04 09:12:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:12:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243957082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:12:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:12:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:12:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:12:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:55.661677+00:00 (in 17.997707 seconds) gcworker stdout | 2025-11-04 09:12:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:07 GMT)" (scheduled at 2025-11-04 09:12:37.663502+00:00) gcworker stdout | 2025-11-04 09:12:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:12:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247257671, None, 1, 0]) gcworker stdout | 2025-11-04 09:12:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:47.766253+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 09:12:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:47 GMT)" (scheduled at 2025-11-04 09:12:37.766253+00:00) notificationworker stdout | 2025-11-04 09:12:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 37, 766997), True, datetime.datetime(2025, 11, 4, 9, 12, 37, 766997), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:12:38,296 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:12:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:12:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:12.482065+00:00 (in 34.001057 seconds) exportactionlogsworker stdout | 2025-11-04 09:12:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:38 GMT)" (scheduled at 2025-11-04 09:12:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:12:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:12:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 38, 481334), True, datetime.datetime(2025, 11, 4, 9, 12, 38, 481334), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:12:38,488 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:12:38,488 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:12:38,488 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:12:39,411 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:47.679968+00:00 (in 8.003238 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:49 GMT)" (scheduled at 2025-11-04 09:12:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:12:39,676 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 39, 676986), True, datetime.datetime(2025, 11, 4, 9, 12, 39, 676986), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:39,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:39,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:39,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:49 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:40,065 [257] [DEBUG] [app] Starting request: urn:request:e08b6ae8-8638-4597-949b-ace5229bcd1c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:12:40,071 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:12:40,073 [250] [DEBUG] [app] Starting request: urn:request:ed6e0dbd-47a7-4678-8425-9dc228a3f922 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:40,074 [250] [DEBUG] [app] Ending request: urn:request:ed6e0dbd-47a7-4678-8425-9dc228a3f922 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ed6e0dbd-47a7-4678-8425-9dc228a3f922', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:12:40,075 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:40,075 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:40,076 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:40,077 [257] [DEBUG] [app] Starting request: urn:request:e3d7c86d-73a9-4d32-920d-1c1a97d3d8a5 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:40,078 [257] [DEBUG] [app] Ending request: urn:request:e3d7c86d-73a9-4d32-920d-1c1a97d3d8a5 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e3d7c86d-73a9-4d32-920d-1c1a97d3d8a5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:40,078 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:40,079 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:40,079 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:40,079 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:40,079 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:40,084 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:40,084 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:40,088 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:40,089 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:40,090 [257] [DEBUG] [app] Ending request: urn:request:e08b6ae8-8638-4597-949b-ace5229bcd1c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e08b6ae8-8638-4597-949b-ace5229bcd1c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:40,090 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:40,090 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.026) securityworker stdout | 2025-11-04 09:12:40,094 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:12:40,100 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:40,123 [254] [DEBUG] [app] Starting request: urn:request:6695b6c5-4c65-4e6b-bde8-b292d515dd3c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:12:40,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:12:40,126 [259] [DEBUG] [app] Starting request: urn:request:b4163302-f0a8-4429-8e6b-e4ba68b0a9c2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:40,126 [259] [DEBUG] [app] Ending request: urn:request:b4163302-f0a8-4429-8e6b-e4ba68b0a9c2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b4163302-f0a8-4429-8e6b-e4ba68b0a9c2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:40,126 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:12:40,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:40,131 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:40,132 [254] [DEBUG] [app] Starting request: urn:request:71c763ad-610b-46f2-b53e-ac5ef91bf2b9 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:40,133 [254] [DEBUG] [app] Ending request: urn:request:71c763ad-610b-46f2-b53e-ac5ef91bf2b9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:71c763ad-610b-46f2-b53e-ac5ef91bf2b9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:40,133 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:40,133 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:40,134 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:40,134 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:40,134 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:40,138 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:40,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:40,142 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:40,143 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:40,143 [254] [DEBUG] [app] Ending request: urn:request:6695b6c5-4c65-4e6b-bde8-b292d515dd3c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6695b6c5-4c65-4e6b-bde8-b292d515dd3c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:40,144 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.021) gunicorn-web stdout | 2025-11-04 09:12:40,144 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:12:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:12:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:06.265965+00:00 (in 26.084833 seconds) repositorygcworker stdout | 2025-11-04 09:12:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:40 GMT)" (scheduled at 2025-11-04 09:12:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:12:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:12:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 40, 181474), True, datetime.datetime(2025, 11, 4, 9, 12, 40, 181474), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:12:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:12:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:12:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:12:40,908 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:12:41,203 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:12:41,493 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:42,132 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:42,135 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:42,136 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:12:42,595 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:42,990 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:12:43,398 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:12:44,196 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:44,648 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:44,654 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:44,660 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:44,683 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:12:44,695 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:12:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:12:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:14.980431+00:00 (in 29.999587 seconds) buildlogsarchiver stdout | 2025-11-04 09:12:44,980 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:14 GMT)" (scheduled at 2025-11-04 09:12:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:12:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 12, 44, 981105), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:12:44,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:12:44,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:12:44,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:12:46,142 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,210 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,272 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,282 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,283 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,292 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,295 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,296 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:12:46,303 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:49.676299+00:00 (in 1.995890 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:47 GMT)" (scheduled at 2025-11-04 09:12:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:12:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:47,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:57.766253+00:00 (in 9.999176 seconds) notificationworker stdout | 2025-11-04 09:12:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:57 GMT)" (scheduled at 2025-11-04 09:12:47.766253+00:00) notificationworker stdout | 2025-11-04 09:12:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 47, 767351), True, datetime.datetime(2025, 11, 4, 9, 12, 47, 767351), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:47,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:47,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:47,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:12:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:12:59.676299+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:59 GMT)" (scheduled at 2025-11-04 09:12:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 49, 677498), True, datetime.datetime(2025, 11, 4, 9, 12, 49, 677498), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:49,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:49,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:12:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:12:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:12:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:01.707296+00:00 (in 12.000422 seconds) securityworker stdout | 2025-11-04 09:12:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:19 GMT)" (scheduled at 2025-11-04 09:12:49.706478+00:00) securityworker stdout | 2025-11-04 09:12:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:12:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:12:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:12:49,709 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:12:49,714 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:49,715 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,715 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 15, 25]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stdout | 2025-11-04 09:12:49,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 27, 37]) securityworker stdout | 2025-11-04 09:12:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:12:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:12:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stdout | 2025-11-04 09:12:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:12:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-27 securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-27 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:12:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 25, 35]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 15-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stdout | 2025-11-04 09:12:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-15 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 5 securityworker stdout | 2025-11-04 09:12:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 4, 14]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:12:49,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:12:49,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-15 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-15 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:12:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 14, 24]) securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-37 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-37 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Left range 4-37 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stdout | 2025-11-04 09:12:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 38, 48]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:12:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:12:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Left range 4-37 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stdout | 2025-11-04 09:12:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 37, 47]) securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-37 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-37 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-48 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Right range 4-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 4-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:12:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:12:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:12:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:12:49,729 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-48 securityworker stdout | 2025-11-04 09:12:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 48 securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 48-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 48-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:12:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 48, 49]) securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 48 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 49-48 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-48 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 36, 46]) securityworker stdout | 2025-11-04 09:12:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Left range 36-46 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stdout | 2025-11-04 09:12:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 46, 49]) securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 36-46 securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] Already merged with block 36-46 securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 36-46 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-36 securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:12:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 12, 22]) securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 1-36 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 12-22 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stdout | 2025-11-04 09:12:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-12 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 2 securityworker stdout | 2025-11-04 09:12:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 1, 11]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-36 securityworker stdout | 2025-11-04 09:12:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:12:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 23, 33]) securityworker stdout | 2025-11-04 09:12:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Right range 23-33 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-23 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Right range 23-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 22-23 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:12:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 22, 32]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:12:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:12:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:12:49,741 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-22 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-22 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-33 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-22 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-33 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 12-33 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Left range 12-33 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-36 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 12-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-36 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stdout | 2025-11-04 09:12:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 33, 43]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-33 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-33 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 12 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 12-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 12-33 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 12 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 11-12 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-12 securityworker stdout | 2025-11-04 09:12:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-12 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-12 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:12:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 11, 21]) securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:12:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 25, 35]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 35-49 securityworker stdout | 2025-11-04 09:12:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:12:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 35, 45]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-45 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Left range 25-45 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:12:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-35 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 25-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 25-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 45, 49]) securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-45 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 25-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 25-45 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 25 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 25 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,750 [92] [DEBUG] [util.migrate.allocator] Total range: 1-25 securityworker stdout | 2025-11-04 09:12:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stdout | 2025-11-04 09:12:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:12:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 10, 20]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 1-25 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-25 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:12:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:12:49,753 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 20, 30]) securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-20 securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-20 securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 10 securityworker stdout | 2025-11-04 09:12:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-20 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-20 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 10 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-10 securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 1-10 securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:12:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 7, 49, 709565), 1, 11]) securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:12:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:12:49,756 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:12:49,756 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:12:51,001 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:00.086435+00:00 (in 7.925540 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,161 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:52 GMT)" (scheduled at 2025-11-04 09:12:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,169 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,169 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:12:52,169 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:12:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:12:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:33.561829+00:00 (in 40.001229 seconds) namespacegcworker stdout | 2025-11-04 09:12:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:53 GMT)" (scheduled at 2025-11-04 09:12:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:12:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:12:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 53, 560963), True, datetime.datetime(2025, 11, 4, 9, 12, 53, 560963), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:12:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:12:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:12:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:13:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:12:55,065 [254] [DEBUG] [app] Starting request: urn:request:97802ecd-646e-40c5-8ec2-431ab38bb4f1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:12:55,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:12:55,068 [250] [DEBUG] [app] Starting request: urn:request:c6b5f1d4-f356-4880-b9df-1a00186971fc (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:55,070 [250] [DEBUG] [app] Ending request: urn:request:c6b5f1d4-f356-4880-b9df-1a00186971fc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c6b5f1d4-f356-4880-b9df-1a00186971fc', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:12:55,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:55,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:55,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:55,073 [254] [DEBUG] [app] Starting request: urn:request:e5cee2f1-dc42-4eae-bf27-826cc7b72b19 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:55,073 [254] [DEBUG] [app] Ending request: urn:request:e5cee2f1-dc42-4eae-bf27-826cc7b72b19 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e5cee2f1-dc42-4eae-bf27-826cc7b72b19', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:12:55,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:12:55,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:55,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:55,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:55,074 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:55,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:55,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:55,084 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:55,085 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:55,086 [254] [DEBUG] [app] Ending request: urn:request:97802ecd-646e-40c5-8ec2-431ab38bb4f1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:97802ecd-646e-40c5-8ec2-431ab38bb4f1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:55,086 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:12:55,086 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.022) gunicorn-web stdout | 2025-11-04 09:12:55,125 [254] [DEBUG] [app] Starting request: urn:request:2504c2ea-bf5c-43d5-ad2d-7fe8211bda1e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:12:55,127 [250] [DEBUG] [app] Starting request: urn:request:a2793652-f225-4bf2-afb5-5b5a0fccdf73 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:12:55,127 [250] [DEBUG] [app] Ending request: urn:request:a2793652-f225-4bf2-afb5-5b5a0fccdf73 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a2793652-f225-4bf2-afb5-5b5a0fccdf73', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:12:55,128 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:12:55,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:55,129 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:12:55,130 [261] [DEBUG] [app] Starting request: urn:request:84c4a9de-f901-45ff-b8e8-9675b05a1af3 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:12:55,131 [261] [DEBUG] [app] Ending request: urn:request:84c4a9de-f901-45ff-b8e8-9675b05a1af3 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:84c4a9de-f901-45ff-b8e8-9675b05a1af3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:12:55,131 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:12:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:12:55,131 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:12:55,132 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:12:55,132 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:12:55,132 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:12:55,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:12:55,136 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:12:55,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:12:55,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:12:55,142 [254] [DEBUG] [app] Ending request: urn:request:2504c2ea-bf5c-43d5-ad2d-7fe8211bda1e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2504c2ea-bf5c-43d5-ad2d-7fe8211bda1e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:12:55,142 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:12:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.018) gunicorn-web stdout | 2025-11-04 09:12:55,142 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:12:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gcworker stdout | 2025-11-04 09:12:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:12:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:07.663502+00:00 (in 12.001399 seconds) gcworker stdout | 2025-11-04 09:12:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:25 GMT)" (scheduled at 2025-11-04 09:12:55.661677+00:00) gcworker stdout | 2025-11-04 09:12:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761037975662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:12:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:12:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:12:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:12:55,709 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:12:56,115 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:12:56,203 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:12:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:12:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) gunicorn-secscan stdout | 2025-11-04 09:12:56,589 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:12:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:12:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) notificationworker stdout | 2025-11-04 09:12:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:12:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:07.766253+00:00 (in 9.999577 seconds) notificationworker stdout | 2025-11-04 09:12:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:07 GMT)" (scheduled at 2025-11-04 09:12:57.766253+00:00) notificationworker stdout | 2025-11-04 09:12:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:12:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 57, 767023), True, datetime.datetime(2025, 11, 4, 9, 12, 57, 767023), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:12:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:12:57,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:12:57,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:12:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:12:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:09.676299+00:00 (in 9.999578 seconds) proxycacheblobworker stdout | 2025-11-04 09:12:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:09 GMT)" (scheduled at 2025-11-04 09:12:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:12:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:12:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 12, 59, 677567), True, datetime.datetime(2025, 11, 4, 9, 12, 59, 677567), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:12:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:12:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:12:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:12:59,917 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:13:00,028 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:52.159714+00:00 (in 52.072760 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:00 GMT)" (scheduled at 2025-11-04 09:13:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:13:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:00 GMT)" executed successfully builder stdout | 2025-11-04 09:13:00,302 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:13:01,112 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:13:01,311 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:19.706478+00:00 (in 17.998742 seconds) securityworker stdout | 2025-11-04 09:13:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:31 GMT)" (scheduled at 2025-11-04 09:13:01.707296+00:00) securityworker stdout | 2025-11-04 09:13:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:01,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:01,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:01,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:01,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:01,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:13:01,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 1, 713651), 1, 49]) securityworker stdout | 2025-11-04 09:13:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:01,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:01,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 1, 713651), 1, 49]) securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:01,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:01,728 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:13:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:13:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:32.081579+00:00 (in 29.999573 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:13:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:32 GMT)" (scheduled at 2025-11-04 09:13:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:13:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:13:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:13:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:13:02,506 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:13:04,110 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:13:04,208 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:13:05,300 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:13:05,796 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:13:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:13:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:40.180734+00:00 (in 33.914365 seconds) repositorygcworker stdout | 2025-11-04 09:13:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:06 GMT)" (scheduled at 2025-11-04 09:13:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:13:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:13:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:13:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:13:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:33.568037+00:00 (in 26.998029 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:13:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:06 GMT)" (scheduled at 2025-11-04 09:13:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:13:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:13:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:13:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:13:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:37.077358+00:00 (in 29.999552 seconds) autopruneworker stdout | 2025-11-04 09:13:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:37 GMT)" (scheduled at 2025-11-04 09:13:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:13:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762243987082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:13:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:13:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:13:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:13:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:25.661677+00:00 (in 17.997760 seconds) gcworker stdout | 2025-11-04 09:13:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:37 GMT)" (scheduled at 2025-11-04 09:13:07.663502+00:00) gcworker stdout | 2025-11-04 09:13:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:13:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247287670, None, 1, 0]) gcworker stdout | 2025-11-04 09:13:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:17.766253+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:13:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:17 GMT)" (scheduled at 2025-11-04 09:13:07.766253+00:00) notificationworker stdout | 2025-11-04 09:13:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 7, 767006), True, datetime.datetime(2025, 11, 4, 9, 13, 7, 767006), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:13:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:13:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.770322+00:00 (in 59.999468 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:13:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:13:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:13:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:13:07,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:13:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:13:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:13:08,309 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:13:09,424 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:19.676299+00:00 (in 9.999567 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:19 GMT)" (scheduled at 2025-11-04 09:13:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 9, 677300), True, datetime.datetime(2025, 11, 4, 9, 13, 9, 677300), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:13:09,690 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:10,066 [257] [DEBUG] [app] Starting request: urn:request:4680fd98-2429-40a1-aec0-f1f0db632ac4 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:10,067 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:10,070 [250] [DEBUG] [app] Starting request: urn:request:e5f19923-3d16-4c63-845e-45abad96bcd0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:10,071 [250] [DEBUG] [app] Ending request: urn:request:e5f19923-3d16-4c63-845e-45abad96bcd0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e5f19923-3d16-4c63-845e-45abad96bcd0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:10,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:10,071 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:10,072 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:10,074 [254] [DEBUG] [app] Starting request: urn:request:66faab29-4383-4ca1-86da-3d994c2670ab (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:10,074 [254] [DEBUG] [app] Ending request: urn:request:66faab29-4383-4ca1-86da-3d994c2670ab (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:66faab29-4383-4ca1-86da-3d994c2670ab', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:10,075 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:10,075 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:10,075 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:10,075 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:10,075 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:10,080 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:10,080 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:10,084 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:10,086 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:10,086 [257] [DEBUG] [app] Ending request: urn:request:4680fd98-2429-40a1-aec0-f1f0db632ac4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4680fd98-2429-40a1-aec0-f1f0db632ac4', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:10,086 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:10,087 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.021) securityworker stdout | 2025-11-04 09:13:10,107 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:13:10,114 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:10,122 [254] [DEBUG] [app] Starting request: urn:request:9590da40-3f4a-4c1f-891d-b4ae4c8755bc (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:10,123 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:10,125 [250] [DEBUG] [app] Starting request: urn:request:c62380de-51ac-4452-a1dd-f5ff90eae337 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:10,125 [250] [DEBUG] [app] Ending request: urn:request:c62380de-51ac-4452-a1dd-f5ff90eae337 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c62380de-51ac-4452-a1dd-f5ff90eae337', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:10,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:10,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:10,130 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:10,132 [261] [DEBUG] [app] Starting request: urn:request:70f7610c-9cf4-46ad-af52-3f251eb1f4e6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:10,132 [261] [DEBUG] [app] Ending request: urn:request:70f7610c-9cf4-46ad-af52-3f251eb1f4e6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:70f7610c-9cf4-46ad-af52-3f251eb1f4e6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:10,133 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:10,133 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:10,133 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:10,133 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:10,133 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:10,137 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:10,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:10,142 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:10,143 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:10,143 [254] [DEBUG] [app] Ending request: urn:request:9590da40-3f4a-4c1f-891d-b4ae4c8755bc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9590da40-3f4a-4c1f-891d-b4ae4c8755bc', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:10,144 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:10,144 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.022) quotaregistrysizeworker stdout | 2025-11-04 09:13:10,920 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:13:11,215 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:13:11,505 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:12,151 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:12,151 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:12,156 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:13:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:13:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:38.480592+00:00 (in 25.998077 seconds) exportactionlogsworker stdout | 2025-11-04 09:13:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:12 GMT)" (scheduled at 2025-11-04 09:13:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:13:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:13:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:13:12,608 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:13,002 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:13:13,411 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:14,210 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:14,670 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:14,662 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:14,682 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:14,698 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:14,710 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:13:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:44.980431+00:00 (in 29.999581 seconds) buildlogsarchiver stdout | 2025-11-04 09:13:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:44 GMT)" (scheduled at 2025-11-04 09:13:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:13:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 13, 14, 981244), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:13:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:13:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:13:16,161 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,225 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,286 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,296 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,300 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,305 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,310 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,312 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:16,323 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:27.766253+00:00 (in 9.999582 seconds) notificationworker stdout | 2025-11-04 09:13:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:27 GMT)" (scheduled at 2025-11-04 09:13:17.766253+00:00) notificationworker stdout | 2025-11-04 09:13:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 17, 767010), True, datetime.datetime(2025, 11, 4, 9, 13, 17, 767010), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:13:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:29.676299+00:00 (in 9.999541 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:29 GMT)" (scheduled at 2025-11-04 09:13:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 19, 677519), True, datetime.datetime(2025, 11, 4, 9, 13, 19, 677519), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:19,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:19,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:19,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:13:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:31.707296+00:00 (in 12.000399 seconds) securityworker stdout | 2025-11-04 09:13:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:49 GMT)" (scheduled at 2025-11-04 09:13:19.706478+00:00) securityworker stdout | 2025-11-04 09:13:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:13:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:13:21,015 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:25,065 [257] [DEBUG] [app] Starting request: urn:request:bb887610-f896-4193-9756-3e54d5173530 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:25,066 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:25,068 [250] [DEBUG] [app] Starting request: urn:request:d74340b9-18af-44f3-87b1-81e21761c8ac (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:25,068 [250] [DEBUG] [app] Ending request: urn:request:d74340b9-18af-44f3-87b1-81e21761c8ac (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d74340b9-18af-44f3-87b1-81e21761c8ac', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:25,068 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:25,069 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:25,070 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:25,076 [261] [DEBUG] [app] Starting request: urn:request:d5f39720-28ff-40eb-9895-f901f043e044 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:25,077 [261] [DEBUG] [app] Ending request: urn:request:d5f39720-28ff-40eb-9895-f901f043e044 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d5f39720-28ff-40eb-9895-f901f043e044', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:25,077 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:25,077 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:25,078 [257] [DEBUG] [data.model.health] Validating database connection. nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:13:25,078 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:25,078 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:25,082 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:25,082 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:25,086 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:25,087 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:25,088 [257] [DEBUG] [app] Ending request: urn:request:bb887610-f896-4193-9756-3e54d5173530 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bb887610-f896-4193-9756-3e54d5173530', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:25,088 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:25,088 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.023) gunicorn-web stdout | 2025-11-04 09:13:25,122 [261] [DEBUG] [app] Starting request: urn:request:8aa9da2b-b8e2-489c-9bdb-617b99ec3e00 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:25,123 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:25,124 [250] [DEBUG] [app] Starting request: urn:request:b891387d-ecbd-4868-8b9b-7f2f5c40f708 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:25,125 [250] [DEBUG] [app] Ending request: urn:request:b891387d-ecbd-4868-8b9b-7f2f5c40f708 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b891387d-ecbd-4868-8b9b-7f2f5c40f708', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:25,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:25,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:25,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:25,129 [258] [DEBUG] [app] Starting request: urn:request:bd20ffe9-2d84-4653-b610-9d544fdfc70c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:25,129 [258] [DEBUG] [app] Ending request: urn:request:bd20ffe9-2d84-4653-b610-9d544fdfc70c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bd20ffe9-2d84-4653-b610-9d544fdfc70c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:25,129 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:25,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:25,130 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:25,130 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:25,130 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:25,134 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:25,134 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:25,138 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:25,140 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:25,141 [261] [DEBUG] [app] Ending request: urn:request:8aa9da2b-b8e2-489c-9bdb-617b99ec3e00 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8aa9da2b-b8e2-489c-9bdb-617b99ec3e00', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:25,141 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:25,141 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gcworker stdout | 2025-11-04 09:13:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:37.663502+00:00 (in 12.001416 seconds) gcworker stdout | 2025-11-04 09:13:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:55 GMT)" (scheduled at 2025-11-04 09:13:25.661677+00:00) gcworker stdout | 2025-11-04 09:13:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038005662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:13:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:13:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:13:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:25,723 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:26,128 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:27.768179+00:00 (in 0.001493 seconds) notificationworker stdout | 2025-11-04 09:13:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:37 GMT)" (scheduled at 2025-11-04 09:13:27.766253+00:00) notificationworker stdout | 2025-11-04 09:13:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 27, 766968), True, datetime.datetime(2025, 11, 4, 9, 13, 27, 766968), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:37.766253+00:00 (in 9.997706 seconds) notificationworker stdout | 2025-11-04 09:13:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:27 GMT)" (scheduled at 2025-11-04 09:13:27.768179+00:00) notificationworker stdout | 2025-11-04 09:13:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:13:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:27,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:27,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:27,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:39.676299+00:00 (in 9.999550 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:39 GMT)" (scheduled at 2025-11-04 09:13:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 29, 677579), True, datetime.datetime(2025, 11, 4, 9, 13, 29, 677579), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:13:29,931 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:13:30,040 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:13:30,315 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:13:31,125 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:13:31,323 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:13:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:49.706478+00:00 (in 17.998730 seconds) securityworker stdout | 2025-11-04 09:13:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:01 GMT)" (scheduled at 2025-11-04 09:13:31.707296+00:00) securityworker stdout | 2025-11-04 09:13:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:31,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 31, 712746), 1, 49]) securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:31,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 31, 712746), 1, 49]) securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:31,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:13:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:31,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:13:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:13:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:02.081579+00:00 (in 29.999589 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:13:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:02 GMT)" (scheduled at 2025-11-04 09:13:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:13:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:13:32,088 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:13:32,088 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:13:32,519 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:13:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:53.560131+00:00 (in 19.997827 seconds) namespacegcworker stdout | 2025-11-04 09:13:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" (scheduled at 2025-11-04 09:13:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:13:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:13:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:13:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:13:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:06.569529+00:00 (in 33.001032 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:13:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" (scheduled at 2025-11-04 09:13:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:13:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:13:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 33, 568830), True, datetime.datetime(2025, 11, 4, 9, 13, 33, 568830), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:13:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:13:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:13:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:13:34,124 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:13:34,223 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:13:35,312 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:13:35,809 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:13:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:13:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.077358+00:00 (in 29.999563 seconds) autopruneworker stdout | 2025-11-04 09:13:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:13:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:13:37,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244017081, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:13:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:13:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:13:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:13:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:55.661677+00:00 (in 17.997756 seconds) gcworker stdout | 2025-11-04 09:13:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:13:37.663502+00:00) gcworker stdout | 2025-11-04 09:13:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:13:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247317671, None, 1, 0]) gcworker stdout | 2025-11-04 09:13:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:37,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:47.766253+00:00 (in 9.999113 seconds) notificationworker stdout | 2025-11-04 09:13:37,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:47 GMT)" (scheduled at 2025-11-04 09:13:37.766253+00:00) notificationworker stdout | 2025-11-04 09:13:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 37, 767486), True, datetime.datetime(2025, 11, 4, 9, 13, 37, 767486), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:13:38,322 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:13:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:13:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:12.482065+00:00 (in 34.001054 seconds) exportactionlogsworker stdout | 2025-11-04 09:13:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:38 GMT)" (scheduled at 2025-11-04 09:13:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:13:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:13:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 38, 481343), True, datetime.datetime(2025, 11, 4, 9, 13, 38, 481343), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:13:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:13:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:13:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:13:39,436 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:47.679968+00:00 (in 8.003251 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:49 GMT)" (scheduled at 2025-11-04 09:13:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 39, 677440), True, datetime.datetime(2025, 11, 4, 9, 13, 39, 677440), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:39,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:39,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:13:39,702 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:40,065 [254] [DEBUG] [app] Starting request: urn:request:d5576336-7744-481b-a966-9439bd7520b4 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:40,068 [256] [DEBUG] [app] Starting request: urn:request:77211690-e47a-43c2-b961-ce284a4ffee2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:40,068 [256] [DEBUG] [app] Ending request: urn:request:77211690-e47a-43c2-b961-ce284a4ffee2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77211690-e47a-43c2-b961-ce284a4ffee2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:40,068 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:13:40,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:40,069 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:40,072 [261] [DEBUG] [app] Starting request: urn:request:25ad07d3-bcca-40f8-9f35-ad8f03b25067 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:40,072 [261] [DEBUG] [app] Ending request: urn:request:25ad07d3-bcca-40f8-9f35-ad8f03b25067 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:25ad07d3-bcca-40f8-9f35-ad8f03b25067', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:40,072 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:40,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:13:40,073 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:40,073 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:40,073 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:40,078 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:40,078 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:40,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:40,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:40,084 [254] [DEBUG] [app] Ending request: urn:request:d5576336-7744-481b-a966-9439bd7520b4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d5576336-7744-481b-a966-9439bd7520b4', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:40,084 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:40,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) securityworker stdout | 2025-11-04 09:13:40,120 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:40,122 [254] [DEBUG] [app] Starting request: urn:request:cec43aa6-f927-4ad5-a934-5391b81f1519 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:13:40,123 [260] [DEBUG] [app] Starting request: urn:request:38c31a41-ecd8-452f-ac3a-c9bcffe06f38 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:40,124 [260] [DEBUG] [app] Ending request: urn:request:38c31a41-ecd8-452f-ac3a-c9bcffe06f38 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:38c31a41-ecd8-452f-ac3a-c9bcffe06f38', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:40,124 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:40,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:40,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:40,127 [261] [DEBUG] [app] Starting request: urn:request:928e4410-c60e-404b-982d-d784873a62fa (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:40,127 [261] [DEBUG] [app] Ending request: urn:request:928e4410-c60e-404b-982d-d784873a62fa (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:928e4410-c60e-404b-982d-d784873a62fa', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:40,127 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:40,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:13:40,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:40,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:40,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql logrotateworker stdout | 2025-11-04 09:13:40,128 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:40,131 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:40,131 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:40,135 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:40,136 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:40,137 [254] [DEBUG] [app] Ending request: urn:request:cec43aa6-f927-4ad5-a934-5391b81f1519 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:cec43aa6-f927-4ad5-a934-5391b81f1519', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:40,137 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:40,137 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.016 118 0.016) repositorygcworker stdout | 2025-11-04 09:13:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:13:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:06.265965+00:00 (in 26.084800 seconds) repositorygcworker stdout | 2025-11-04 09:13:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:40 GMT)" (scheduled at 2025-11-04 09:13:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:13:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:13:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 40, 181537), True, datetime.datetime(2025, 11, 4, 9, 13, 40, 181537), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:13:40,189 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:13:40,189 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:13:40,189 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:13:40,933 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:13:41,228 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:13:41,519 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:42,166 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:42,173 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:13:42,177 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:13:42,622 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:43,015 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:13:43,424 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:13:44,236 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:44,685 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:44,701 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:44,704 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:44,712 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:13:44,724 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:13:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:14.980431+00:00 (in 29.999586 seconds) buildlogsarchiver stdout | 2025-11-04 09:13:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:14 GMT)" (scheduled at 2025-11-04 09:13:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:13:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 13, 44, 981123), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:13:44,987 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:13:44,987 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:13:44,987 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:13:46,181 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,248 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,306 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,320 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,323 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,326 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,333 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,336 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:13:46,345 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:49.676299+00:00 (in 1.995816 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:47 GMT)" (scheduled at 2025-11-04 09:13:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:13:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:57.766253+00:00 (in 9.999546 seconds) notificationworker stdout | 2025-11-04 09:13:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:57 GMT)" (scheduled at 2025-11-04 09:13:47.766253+00:00) notificationworker stdout | 2025-11-04 09:13:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 47, 767140), True, datetime.datetime(2025, 11, 4, 9, 13, 47, 767140), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:47,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:47,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:47,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:13:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:13:59.676299+00:00 (in 9.999418 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:59 GMT)" (scheduled at 2025-11-04 09:13:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 49, 677279), True, datetime.datetime(2025, 11, 4, 9, 13, 49, 677279), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:49,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:49,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:49,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:13:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:13:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:13:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:01.707296+00:00 (in 12.000374 seconds) securityworker stdout | 2025-11-04 09:13:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:19 GMT)" (scheduled at 2025-11-04 09:13:49.706478+00:00) securityworker stdout | 2025-11-04 09:13:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:13:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:13:49,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:13:49,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:49,719 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:13:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 14, 24]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stdout | 2025-11-04 09:13:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-49 securityworker stdout | 2025-11-04 09:13:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-32 securityworker stdout | 2025-11-04 09:13:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-32 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:13:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 24, 34]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-42 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Left range 14-42 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:13:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:13:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 42, 49]) securityworker stdout | 2025-11-04 09:13:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:13:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:13:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-42 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-42 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 14 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-42 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stdout | 2025-11-04 09:13:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stdout | 2025-11-04 09:13:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 4, 14]) securityworker stdout | 2025-11-04 09:13:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:13:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 4 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 4 securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-4 securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:13:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-4 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:13:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:13:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 14, 24]) securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 14-24 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stdout | 2025-11-04 09:13:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stdout | 2025-11-04 09:13:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 1, 11]) securityworker stdout | 2025-11-04 09:13:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Right range 14-24 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-14 securityworker stdout | 2025-11-04 09:13:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-14 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:13:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 11, 21]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:13:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:13:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:13:49,741 [92] [DEBUG] [util.migrate.allocator] Merging with block 14-24 securityworker stdout | 2025-11-04 09:13:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 24 securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 24-49 securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 14-24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 24 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 24-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 29, 39]) securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 24-49 securityworker stdout | 2025-11-04 09:13:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 29-39 securityworker stdout | 2025-11-04 09:13:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-29 securityworker stdout | 2025-11-04 09:13:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:13:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 24, 34]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 24-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 29-39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-29 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:13:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:13:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:13:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with block 29-39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 29-39 securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 39-49 securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 39-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 39, 49]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 49-39 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:13:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:13:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 38, 48]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,752 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stdout | 2025-11-04 09:13:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-38 securityworker stdout | 2025-11-04 09:13:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-38 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:13:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 23, 33]) securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:13:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:13:49,754 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stdout | 2025-11-04 09:13:49,754 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stdout | 2025-11-04 09:13:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-38 securityworker stdout | 2025-11-04 09:13:49,754 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 23-33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 38-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 33-38 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 33 securityworker stdout | 2025-11-04 09:13:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 33, 43]) securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 23-33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 23-33 securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] Already merged with block 23-33 securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stdout | 2025-11-04 09:13:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 23-33 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 38-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-48 securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 23-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 23-48 securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-23 securityworker stdout | 2025-11-04 09:13:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 23-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-23 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:13:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 5, 15]) securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Left range 5-15 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Right range 23-48 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-23 securityworker stdout | 2025-11-04 09:13:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 5-15 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Right range 23-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-23 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:13:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 15, 25]) securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 5-15 securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 5-15 securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] Already merged with block 5-15 securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 5-15 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 23-48 securityworker stdout | 2025-11-04 09:13:49,761 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-48 securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Left range 5-48 securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Left range 5-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:13:49,762 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:13:49,763 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 48, 49]) securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 5-48 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Already merged with block 5-48 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 5 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 5-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 5-48 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 5 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Total range: 1-5 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:13:49,764 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-5 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:13:49,765 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 8, 49, 712638), 1, 11]) securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:13:49,766 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:13:49,766 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:13:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:13:51,035 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:00.086435+00:00 (in 7.925924 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:52 GMT)" (scheduled at 2025-11-04 09:13:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,169 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,169 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:13:52,169 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:13:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:13:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:33.561829+00:00 (in 40.001076 seconds) namespacegcworker stdout | 2025-11-04 09:13:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:53 GMT)" (scheduled at 2025-11-04 09:13:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:13:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:13:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 53, 561139), True, datetime.datetime(2025, 11, 4, 9, 13, 53, 561139), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:13:53,569 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:13:53,569 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:13:53,569 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:14:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:13:55,067 [254] [DEBUG] [app] Starting request: urn:request:c315b2ee-2c2b-494a-8673-dcf71ab1a8c2 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:13:55,074 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:13:55,076 [259] [DEBUG] [app] Starting request: urn:request:934bf8af-6591-49b9-8b84-4f52ee917ba1 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:55,077 [259] [DEBUG] [app] Ending request: urn:request:934bf8af-6591-49b9-8b84-4f52ee917ba1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:934bf8af-6591-49b9-8b84-4f52ee917ba1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:55,077 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:13:55,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:55,079 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:55,080 [261] [DEBUG] [app] Starting request: urn:request:855f9383-dfa2-4948-9ddb-af63eee93f00 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:55,081 [261] [DEBUG] [app] Ending request: urn:request:855f9383-dfa2-4948-9ddb-af63eee93f00 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:855f9383-dfa2-4948-9ddb-af63eee93f00', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:55,081 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:13:55,082 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:55,082 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:55,083 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:55,083 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:55,088 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:55,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:55,092 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:55,094 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:55,095 [254] [DEBUG] [app] Ending request: urn:request:c315b2ee-2c2b-494a-8673-dcf71ab1a8c2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c315b2ee-2c2b-494a-8673-dcf71ab1a8c2', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:55,095 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:55,096 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.030 118 0.029) gunicorn-web stdout | 2025-11-04 09:13:55,122 [254] [DEBUG] [app] Starting request: urn:request:01e82f30-4dfc-4a52-9cbf-b39350394e4b (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:13:55,124 [260] [DEBUG] [app] Starting request: urn:request:4e8b2fd1-7445-474d-947d-73e540a46322 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:13:55,125 [260] [DEBUG] [app] Ending request: urn:request:4e8b2fd1-7445-474d-947d-73e540a46322 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4e8b2fd1-7445-474d-947d-73e540a46322', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:13:55,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:13:55,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:55,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:13:55,129 [261] [DEBUG] [app] Starting request: urn:request:df6204e7-c713-44b8-9d2b-60f046c9ced6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:13:55,129 [261] [DEBUG] [app] Ending request: urn:request:df6204e7-c713-44b8-9d2b-60f046c9ced6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:df6204e7-c713-44b8-9d2b-60f046c9ced6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:13:55,130 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:13:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:13:55,131 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:13:55,131 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:13:55,131 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:13:55,131 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:13:55,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:13:55,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:13:55,139 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:13:55,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:13:55,142 [254] [DEBUG] [app] Ending request: urn:request:01e82f30-4dfc-4a52-9cbf-b39350394e4b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:01e82f30-4dfc-4a52-9cbf-b39350394e4b', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:13:55,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:13:55,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:13:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:13:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.021) gcworker stdout | 2025-11-04 09:13:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:13:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.663502+00:00 (in 12.001361 seconds) gcworker stdout | 2025-11-04 09:13:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:25 GMT)" (scheduled at 2025-11-04 09:13:55.661677+00:00) gcworker stdout | 2025-11-04 09:13:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038035662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:13:55,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:13:55,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:13:55,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:13:55,740 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:13:56,143 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:13:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:13:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:07.766253+00:00 (in 9.999563 seconds) notificationworker stdout | 2025-11-04 09:13:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:13:57.766253+00:00) notificationworker stdout | 2025-11-04 09:13:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:13:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 57, 767156), True, datetime.datetime(2025, 11, 4, 9, 13, 57, 767156), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:13:57,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:13:57,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:13:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:13:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:13:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:09.676299+00:00 (in 9.999508 seconds) proxycacheblobworker stdout | 2025-11-04 09:13:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:09 GMT)" (scheduled at 2025-11-04 09:13:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:13:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:13:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 13, 59, 677107), True, datetime.datetime(2025, 11, 4, 9, 13, 59, 677107), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:13:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:13:59,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:13:59,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:13:59,947 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:14:00,056 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:52.159714+00:00 (in 52.072763 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:00 GMT)" (scheduled at 2025-11-04 09:14:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,095 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:14:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:00 GMT)" executed successfully builder stdout | 2025-11-04 09:14:00,333 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:14:01,144 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:14:01,339 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:14:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:19.706478+00:00 (in 17.998675 seconds) securityworker stdout | 2025-11-04 09:14:01,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:31 GMT)" (scheduled at 2025-11-04 09:14:01.707296+00:00) securityworker stdout | 2025-11-04 09:14:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:01,712 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:01,715 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:01,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:01,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 1, 715180), 1, 49]) securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:01,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:01,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:01,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:01,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:01,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:01,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 1, 715180), 1, 49]) securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:01,733 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:14:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:01,733 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:14:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:14:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:32.081579+00:00 (in 29.999561 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:14:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:32 GMT)" (scheduled at 2025-11-04 09:14:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:14:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:14:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:14:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:14:02,535 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:14:04,141 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:14:04,240 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:14:05,330 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:14:05,826 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:06,267 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:40.180734+00:00 (in 33.913593 seconds) repositorygcworker stdout | 2025-11-04 09:14:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:06 GMT)" (scheduled at 2025-11-04 09:14:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:14:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:14:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:14:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:14:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:33.568037+00:00 (in 26.997811 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:14:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:06 GMT)" (scheduled at 2025-11-04 09:14:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:14:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:14:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:14:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:14:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:37.077358+00:00 (in 29.999554 seconds) autopruneworker stdout | 2025-11-04 09:14:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:37 GMT)" (scheduled at 2025-11-04 09:14:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:14:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244047083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:14:07,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:14:07,088 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:14:07,088 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:37 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:14:07,581 [67] [DEBUG] [apscheduler.scheduler] Looking for jobs to run globalpromstats stdout | 2025-11-04 09:14:07,582 [67] [INFO] [apscheduler.executors.default] Running job "GlobalPrometheusStatsWorker._try_report_stats (trigger: interval[1:00:00], next run at: 2025-11-04 09:14:07 GMT)" (scheduled at 2025-11-04 09:14:07.580640+00:00) globalpromstats stdout | 2025-11-04 09:14:07,582 [67] [DEBUG] [__main__] Attempting to report stats globalpromstats stdout | 2025-11-04 09:14:07,582 [67] [DEBUG] [util.locking] Acquiring global lock GLOBAL_PROM_STATS globalpromstats stdout | 2025-11-04 09:14:07,582 [67] [DEBUG] [redis_lock.acquire] Getting 'lock:GLOBAL_PROM_STATS' ... globalpromstats stdout | 2025-11-04 09:14:07,583 [67] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 10:14:07.580640+00:00 (in 3599.997621 seconds) globalpromstats stdout | 2025-11-04 09:14:07,583 [67] [WARNING] [util.locking] Could not connect to Redis for lock GLOBAL_PROM_STATS: Connection closed by server. globalpromstats stdout | 2025-11-04 09:14:07,583 [67] [DEBUG] [__main__] Could not acquire global lock for global prometheus stats globalpromstats stdout | 2025-11-04 09:14:07,583 [67] [INFO] [apscheduler.executors.default] Job "GlobalPrometheusStatsWorker._try_report_stats (trigger: interval[1:00:00], next run at: 2025-11-04 10:14:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:14:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:25.661677+00:00 (in 17.997680 seconds) gcworker stdout | 2025-11-04 09:14:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:37 GMT)" (scheduled at 2025-11-04 09:14:07.663502+00:00) gcworker stdout | 2025-11-04 09:14:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:14:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247347670, None, 1, 0]) gcworker stdout | 2025-11-04 09:14:07,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:07,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:17.766253+00:00 (in 9.999516 seconds) notificationworker stdout | 2025-11-04 09:14:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:17 GMT)" (scheduled at 2025-11-04 09:14:07.766253+00:00) notificationworker stdout | 2025-11-04 09:14:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 7, 767095), True, datetime.datetime(2025, 11, 4, 9, 14, 7, 767095), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:14:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:14:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:07.770322+00:00 (in 59.999480 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:14:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:07 GMT)" (scheduled at 2025-11-04 09:14:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:14:07,772 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:14:07,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:07,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:07,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:14:07,778 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:14:07,778 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:14:08,337 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:14:09,453 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:19.676299+00:00 (in 9.999482 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:19 GMT)" (scheduled at 2025-11-04 09:14:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 9, 677204), True, datetime.datetime(2025, 11, 4, 9, 14, 9, 677204), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:09,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:09,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:09,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:14:09,720 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:10,066 [254] [DEBUG] [app] Starting request: urn:request:487075c2-c9db-4ea3-81c2-949d89b8127d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:10,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:10,069 [260] [DEBUG] [app] Starting request: urn:request:299ee4c0-96f7-405c-a942-4067110840a8 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:10,070 [260] [DEBUG] [app] Ending request: urn:request:299ee4c0-96f7-405c-a942-4067110840a8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:299ee4c0-96f7-405c-a942-4067110840a8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:14:10,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:10,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:10,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:10,074 [257] [DEBUG] [app] Starting request: urn:request:0d3d08f8-a88a-4c00-80d5-2004423045db (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:10,074 [257] [DEBUG] [app] Ending request: urn:request:0d3d08f8-a88a-4c00-80d5-2004423045db (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0d3d08f8-a88a-4c00-80d5-2004423045db', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:10,074 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:10,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:10,075 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:10,075 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:10,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:10,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:10,080 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:10,085 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:10,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:10,088 [254] [DEBUG] [app] Ending request: urn:request:487075c2-c9db-4ea3-81c2-949d89b8127d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:487075c2-c9db-4ea3-81c2-949d89b8127d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:10,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:10,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) gunicorn-web stdout | 2025-11-04 09:14:10,123 [261] [DEBUG] [app] Starting request: urn:request:f001a91a-b683-4a0d-82bc-b73cffc827fb (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:10,124 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:10,126 [260] [DEBUG] [app] Starting request: urn:request:bd3a6d3a-da46-4c17-8267-9d84649c8af5 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:10,126 [260] [DEBUG] [app] Ending request: urn:request:bd3a6d3a-da46-4c17-8267-9d84649c8af5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bd3a6d3a-da46-4c17-8267-9d84649c8af5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:10,127 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:10,127 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:14:10,128 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:10,130 [254] [DEBUG] [app] Starting request: urn:request:751fda66-d369-44e7-93f6-4f7c9506575e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:10,130 [254] [DEBUG] [app] Ending request: urn:request:751fda66-d369-44e7-93f6-4f7c9506575e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:751fda66-d369-44e7-93f6-4f7c9506575e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:10,131 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:10,131 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:10,132 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:10,132 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:10,132 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:10,137 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:10,137 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) securityworker stdout | 2025-11-04 09:14:10,139 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:10,142 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:10,144 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:10,145 [261] [DEBUG] [app] Ending request: urn:request:f001a91a-b683-4a0d-82bc-b73cffc827fb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f001a91a-b683-4a0d-82bc-b73cffc827fb', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:10,145 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:10,145 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.024) logrotateworker stdout | 2025-11-04 09:14:10,147 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:14:10,949 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:14:11,244 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:14:11,535 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:12,195 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:12,212 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:12,213 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:14:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:14:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:38.480592+00:00 (in 25.998012 seconds) exportactionlogsworker stdout | 2025-11-04 09:14:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:12 GMT)" (scheduled at 2025-11-04 09:14:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:14:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:14:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:14:12,653 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:13,033 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:14:13,441 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:14,253 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:14,706 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:14,721 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:14,731 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:14,740 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:14,720 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:14:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:44.980431+00:00 (in 29.999483 seconds) buildlogsarchiver stdout | 2025-11-04 09:14:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:44 GMT)" (scheduled at 2025-11-04 09:14:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:14:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 14, 14, 981275), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:14,991 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:14:14,991 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:14:14,991 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:14:16,200 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,285 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,329 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,345 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,347 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,348 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,356 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,358 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:16,365 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:27.766253+00:00 (in 9.999505 seconds) notificationworker stdout | 2025-11-04 09:14:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:27 GMT)" (scheduled at 2025-11-04 09:14:17.766253+00:00) notificationworker stdout | 2025-11-04 09:14:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 17, 767193), True, datetime.datetime(2025, 11, 4, 9, 14, 17, 767193), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:17,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:17,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:14:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:29.676299+00:00 (in 9.999456 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:19,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:29 GMT)" (scheduled at 2025-11-04 09:14:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 19, 677292), True, datetime.datetime(2025, 11, 4, 9, 14, 19, 677292), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:14:19,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:19,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:31.707296+00:00 (in 11.999772 seconds) securityworker stdout | 2025-11-04 09:14:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:49 GMT)" (scheduled at 2025-11-04 09:14:19.706478+00:00) securityworker stdout | 2025-11-04 09:14:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:19,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:19,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:19,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:14:19,718 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:14:19,718 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:14:21,053 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:25,065 [254] [DEBUG] [app] Starting request: urn:request:7e42f10c-2cf7-4e88-94b4-036492fec12f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:25,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:25,072 [250] [DEBUG] [app] Starting request: urn:request:a70d196f-3c22-4f6a-9665-18a821964e44 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:25,073 [250] [DEBUG] [app] Ending request: urn:request:a70d196f-3c22-4f6a-9665-18a821964e44 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a70d196f-3c22-4f6a-9665-18a821964e44', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:25,073 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:25,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:14:25,074 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:25,076 [254] [DEBUG] [app] Starting request: urn:request:5466ce90-99c6-4b61-8d17-5438a6770982 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:25,077 [254] [DEBUG] [app] Ending request: urn:request:5466ce90-99c6-4b61-8d17-5438a6770982 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5466ce90-99c6-4b61-8d17-5438a6770982', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:25,077 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:25,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:25,078 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:25,078 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:25,078 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:25,087 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:25,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:25,092 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:25,094 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:25,095 [254] [DEBUG] [app] Ending request: urn:request:7e42f10c-2cf7-4e88-94b4-036492fec12f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7e42f10c-2cf7-4e88-94b4-036492fec12f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:25,095 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:25,095 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.031 118 0.031) gunicorn-web stdout | 2025-11-04 09:14:25,122 [257] [DEBUG] [app] Starting request: urn:request:048a78c0-583c-4465-8779-dfd8b87b5129 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:25,123 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:25,125 [250] [DEBUG] [app] Starting request: urn:request:8f06a9a8-3a25-4b51-9e94-1be24fea67fd (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:25,126 [250] [DEBUG] [app] Ending request: urn:request:8f06a9a8-3a25-4b51-9e94-1be24fea67fd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8f06a9a8-3a25-4b51-9e94-1be24fea67fd', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:25,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:14:25,126 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:25,128 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:25,129 [254] [DEBUG] [app] Starting request: urn:request:0471d5b1-f15a-40a3-acea-73f22ede36bf (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:25,130 [254] [DEBUG] [app] Ending request: urn:request:0471d5b1-f15a-40a3-acea-73f22ede36bf (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0471d5b1-f15a-40a3-acea-73f22ede36bf', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:25,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:25,130 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:25,131 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:25,131 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:25,131 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:25,135 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:25,135 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:25,140 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:25,142 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:25,143 [257] [DEBUG] [app] Ending request: urn:request:048a78c0-583c-4465-8779-dfd8b87b5129 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:048a78c0-583c-4465-8779-dfd8b87b5129', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:25,143 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:25,144 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.022) gcworker stdout | 2025-11-04 09:14:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:37.663502+00:00 (in 12.001293 seconds) gcworker stdout | 2025-11-04 09:14:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:55 GMT)" (scheduled at 2025-11-04 09:14:25.661677+00:00) gcworker stdout | 2025-11-04 09:14:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038065662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:14:25,674 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:14:25,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:25,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:14:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:25,771 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:26,159 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:27.768179+00:00 (in 0.001438 seconds) notificationworker stdout | 2025-11-04 09:14:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:37 GMT)" (scheduled at 2025-11-04 09:14:27.766253+00:00) notificationworker stdout | 2025-11-04 09:14:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 27, 767062), True, datetime.datetime(2025, 11, 4, 9, 14, 27, 767062), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:37.766253+00:00 (in 9.997627 seconds) notificationworker stdout | 2025-11-04 09:14:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:27 GMT)" (scheduled at 2025-11-04 09:14:27.768179+00:00) notificationworker stdout | 2025-11-04 09:14:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:14:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:39.676299+00:00 (in 9.999434 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:29,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:39 GMT)" (scheduled at 2025-11-04 09:14:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 29, 677281), True, datetime.datetime(2025, 11, 4, 9, 14, 29, 677281), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:29,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:14:29,965 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:14:30,073 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:14:30,368 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:14:31,177 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:14:31,355 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:14:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:49.706478+00:00 (in 17.998609 seconds) securityworker stdout | 2025-11-04 09:14:31,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:01 GMT)" (scheduled at 2025-11-04 09:14:31.707296+00:00) securityworker stdout | 2025-11-04 09:14:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:31,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:31,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:31,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:31,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 31, 714613), 1, 49]) securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:31,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:31,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 31, 714613), 1, 49]) securityworker stdout | 2025-11-04 09:14:31,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:14:31,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:14:31,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:14:31,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:31,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:31,731 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:14:31,731 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:31,731 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:14:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:31,731 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:14:32,082 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:14:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:02.081579+00:00 (in 29.999207 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:14:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:02 GMT)" (scheduled at 2025-11-04 09:14:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:14:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:14:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:14:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:14:32,552 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:14:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:53.560131+00:00 (in 19.997744 seconds) namespacegcworker stdout | 2025-11-04 09:14:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" (scheduled at 2025-11-04 09:14:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:14:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:14:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:14:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:14:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:06.569529+00:00 (in 33.001056 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:14:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" (scheduled at 2025-11-04 09:14:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:14:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:14:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 33, 568899), True, datetime.datetime(2025, 11, 4, 9, 14, 33, 568899), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:14:33,578 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:14:33,578 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:14:33,578 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:14:34,160 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:14:34,273 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:14:35,348 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:14:35,845 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:14:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:14:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:07.077358+00:00 (in 29.999500 seconds) autopruneworker stdout | 2025-11-04 09:14:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:07 GMT)" (scheduled at 2025-11-04 09:14:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:14:37,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244077083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:14:37,088 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:14:37,088 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:14:37,088 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:14:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:55.661677+00:00 (in 17.997670 seconds) gcworker stdout | 2025-11-04 09:14:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:07 GMT)" (scheduled at 2025-11-04 09:14:37.663502+00:00) gcworker stdout | 2025-11-04 09:14:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:14:37,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247377671, None, 1, 0]) gcworker stdout | 2025-11-04 09:14:37,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:37,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:47.766253+00:00 (in 9.999565 seconds) notificationworker stdout | 2025-11-04 09:14:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:47 GMT)" (scheduled at 2025-11-04 09:14:37.766253+00:00) notificationworker stdout | 2025-11-04 09:14:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 37, 767025), True, datetime.datetime(2025, 11, 4, 9, 14, 37, 767025), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:37,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:37,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:37,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:14:38,355 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:14:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:14:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:12.482065+00:00 (in 34.000941 seconds) exportactionlogsworker stdout | 2025-11-04 09:14:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:38 GMT)" (scheduled at 2025-11-04 09:14:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:14:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:14:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 38, 481508), True, datetime.datetime(2025, 11, 4, 9, 14, 38, 481508), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:14:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:14:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:14:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:14:39,469 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:44.679698+00:00 (in 5.002815 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:39,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:49 GMT)" (scheduled at 2025-11-04 09:14:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 39, 677316), True, datetime.datetime(2025, 11, 4, 9, 14, 39, 677316), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:39,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:39,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:39,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:14:39,736 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:40,065 [254] [DEBUG] [app] Starting request: urn:request:f81e7217-7f1e-4db2-8a72-cd8ff7187768 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:40,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:40,069 [260] [DEBUG] [app] Starting request: urn:request:b874206e-1644-452e-9f2c-72eb14e1b36a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:40,069 [260] [DEBUG] [app] Ending request: urn:request:b874206e-1644-452e-9f2c-72eb14e1b36a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b874206e-1644-452e-9f2c-72eb14e1b36a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:14:40,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:40,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:40,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:40,073 [261] [DEBUG] [app] Starting request: urn:request:d8ad322c-0186-4432-a1c2-1b62f503fbd8 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:40,074 [261] [DEBUG] [app] Ending request: urn:request:d8ad322c-0186-4432-a1c2-1b62f503fbd8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d8ad322c-0186-4432-a1c2-1b62f503fbd8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:40,074 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:40,075 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:40,075 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:40,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:40,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:40,080 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:40,085 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:40,086 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:40,088 [254] [DEBUG] [app] Ending request: urn:request:f81e7217-7f1e-4db2-8a72-cd8ff7187768 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f81e7217-7f1e-4db2-8a72-cd8ff7187768', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:40,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:40,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.024) gunicorn-web stdout | 2025-11-04 09:14:40,122 [254] [DEBUG] [app] Starting request: urn:request:fb4f2788-49e1-471f-8bb0-2cede2a0ff5a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:14:40,124 [260] [DEBUG] [app] Starting request: urn:request:9f30653c-7dca-4c7a-9bcd-6f0cdce51639 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:40,124 [260] [DEBUG] [app] Ending request: urn:request:9f30653c-7dca-4c7a-9bcd-6f0cdce51639 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:9f30653c-7dca-4c7a-9bcd-6f0cdce51639', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:14:40,124 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:40,124 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:14:40,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:40,127 [254] [DEBUG] [app] Starting request: urn:request:8c3f15ef-ea7b-4198-b650-3bff0cc552ca (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:40,128 [254] [DEBUG] [app] Ending request: urn:request:8c3f15ef-ea7b-4198-b650-3bff0cc552ca (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8c3f15ef-ea7b-4198-b650-3bff0cc552ca', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:14:40,128 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:40,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:40,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:40,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:40,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:40,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:40,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:40,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:40,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:40,140 [254] [DEBUG] [app] Ending request: urn:request:fb4f2788-49e1-471f-8bb0-2cede2a0ff5a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fb4f2788-49e1-471f-8bb0-2cede2a0ff5a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:40,140 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:40,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) securityworker stdout | 2025-11-04 09:14:40,156 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:14:40,164 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:41.259858+00:00 (in 1.078671 seconds) repositorygcworker stdout | 2025-11-04 09:14:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:40 GMT)" (scheduled at 2025-11-04 09:14:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:14:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:14:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 40, 181594), True, datetime.datetime(2025, 11, 4, 9, 14, 40, 181594), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:14:40,190 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:14:40,190 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:14:40,190 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:40 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:14:40,966 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:41,260 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:14:41,260 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:06.265965+00:00 (in 25.005501 seconds) repositorygcworker stdout | 2025-11-04 09:14:41,260 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:41 GMT)" (scheduled at 2025-11-04 09:14:41.259858+00:00) blobuploadcleanupworker stdout | 2025-11-04 09:14:41,259 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:14:41,261 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:41,269 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), True, datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:41,271 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), True, datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 14, 41, 260826), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:14:41,272 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:14:41,272 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:41 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:14:41,552 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:42,223 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:42,233 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:14:42,234 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:14:42,669 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:43,053 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:14:43,458 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:14:44,270 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:44,679 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:44,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:47.679968+00:00 (in 2.999688 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:44,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:44 GMT)" (scheduled at 2025-11-04 09:14:44.679698+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:44,681 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:44,689 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), True, datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:44,691 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), True, datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 14, 44, 680614), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:14:44,694 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:44,694 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:19:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:14:44,725 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:44,741 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:44,748 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:44,758 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:14:44,899 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:14:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:14.980431+00:00 (in 29.999486 seconds) buildlogsarchiver stdout | 2025-11-04 09:14:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:14 GMT)" (scheduled at 2025-11-04 09:14:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:14:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 14, 44, 981263), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:14:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:14:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:14:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:14:46,221 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,308 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,350 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,370 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,373 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,385 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,389 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,398 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:14:46,381 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:49.676299+00:00 (in 1.995709 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:47 GMT)" (scheduled at 2025-11-04 09:14:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:14:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:57.766253+00:00 (in 9.999468 seconds) notificationworker stdout | 2025-11-04 09:14:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:57 GMT)" (scheduled at 2025-11-04 09:14:47.766253+00:00) notificationworker stdout | 2025-11-04 09:14:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 47, 767215), True, datetime.datetime(2025, 11, 4, 9, 14, 47, 767215), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:47,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:47,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:47,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:14:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:14:59.676299+00:00 (in 9.999401 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:59 GMT)" (scheduled at 2025-11-04 09:14:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 49, 677259), True, datetime.datetime(2025, 11, 4, 9, 14, 49, 677259), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:49,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:49,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:49,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:14:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:14:49,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:14:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:01.707296+00:00 (in 11.999985 seconds) securityworker stdout | 2025-11-04 09:14:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:19 GMT)" (scheduled at 2025-11-04 09:14:49.706478+00:00) securityworker stdout | 2025-11-04 09:14:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:14:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:14:49,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:14:49,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:14:49,718 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 33, 43]) securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Left range 33-43 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stdout | 2025-11-04 09:14:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Left range 33-43 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stdout | 2025-11-04 09:14:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 43, 49]) securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 33-43 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 33-43 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Already merged with block 33-43 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 33 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 33-43 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-33 securityworker stdout | 2025-11-04 09:14:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-33 securityworker stdout | 2025-11-04 09:14:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:14:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 22, 32]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:14:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:14:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 9, 19]) securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Right range 9-19 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stdout | 2025-11-04 09:14:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 9-19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-19 securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 19 securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 19-33 securityworker stdout | 2025-11-04 09:14:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,731 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 19-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stdout | 2025-11-04 09:14:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-22 securityworker stdout | 2025-11-04 09:14:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-22 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stdout | 2025-11-04 09:14:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 19, 29]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 32-33 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stdout | 2025-11-04 09:14:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 32-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:14:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 42-32 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 42-32 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 3, 13]) securityworker stdout | 2025-11-04 09:14:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:14:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Right range 3-13 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 3-13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 1, 11]) securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-13 securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stdout | 2025-11-04 09:14:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-49 securityworker stdout | 2025-11-04 09:14:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 35, 45]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:14:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:14:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:14:49,741 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stdout | 2025-11-04 09:14:49,741 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:14:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 45, 49]) securityworker stdout | 2025-11-04 09:14:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:14:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:14:49,743 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stdout | 2025-11-04 09:14:49,743 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stdout | 2025-11-04 09:14:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 13-35 securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:14:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:14:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 19, 29]) securityworker stdout | 2025-11-04 09:14:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:14:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 13-35 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-35 securityworker stdout | 2025-11-04 09:14:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stdout | 2025-11-04 09:14:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 29, 39]) securityworker stdout | 2025-11-04 09:14:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:14:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-29 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-29 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 13-19 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-29 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 13-19 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-19 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-19 securityworker stdout | 2025-11-04 09:14:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:14:49,749 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 13, 23]) securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 13 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 23-13 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-13 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:14:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,753 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 4, 14]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:14:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:14:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:14:49,754 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:14:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:14:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stdout | 2025-11-04 09:14:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 4-14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:14:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 1, 11]) securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 4-14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-49 securityworker stdout | 2025-11-04 09:14:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 15, 25]) securityworker stdout | 2025-11-04 09:14:49,758 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 15-25 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Right range 15-25 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-15 securityworker stdout | 2025-11-04 09:14:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 14-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 15-25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-15 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:14:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 14, 24]) securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 15-25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 25-49 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stdout | 2025-11-04 09:14:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 25-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:14:49,762 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 34, 44]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Total range: 25-49 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-34 securityworker stdout | 2025-11-04 09:14:49,763 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 25-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 25-34 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:14:49,764 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 25, 35]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 25-35 by worker securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 44 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Total range: 44-49 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:14:49,765 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 25-35 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 44-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:14:49,766 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 9, 49, 712204), 44, 49]) securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:14:49,767 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:14:49,767 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:14:49,767 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 44 securityworker stdout | 2025-11-04 09:14:49,767 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:14:49,768 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:14:49,768 [92] [DEBUG] [util.migrate.allocator] Total range: 49-44 securityworker stdout | 2025-11-04 09:14:49,768 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:49,768 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-44 securityworker stderr | 2025-11-04 09:14:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:14:49,768 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:14:51,070 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:00.086435+00:00 (in 7.926189 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:52 GMT)" (scheduled at 2025-11-04 09:14:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,160 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,169 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,169 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:14:52,169 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:14:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:14:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:33.561829+00:00 (in 40.001022 seconds) namespacegcworker stdout | 2025-11-04 09:14:53,561 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:53 GMT)" (scheduled at 2025-11-04 09:14:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:14:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:14:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 53, 561252), True, datetime.datetime(2025, 11, 4, 9, 14, 53, 561252), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:14:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:14:53,570 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:14:53,570 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:15:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:14:55,065 [261] [DEBUG] [app] Starting request: urn:request:136a75ec-5e05-4796-b2f8-ee724db739d0 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:14:55,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:14:55,068 [250] [DEBUG] [app] Starting request: urn:request:4a01d36d-d22e-4ff5-bf19-e21c7cde55f7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:55,069 [250] [DEBUG] [app] Ending request: urn:request:4a01d36d-d22e-4ff5-bf19-e21c7cde55f7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4a01d36d-d22e-4ff5-bf19-e21c7cde55f7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:14:55,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:55,069 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:55,070 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:55,072 [258] [DEBUG] [app] Starting request: urn:request:c1cb7771-5f4e-430d-8e09-8a678c366370 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:55,073 [258] [DEBUG] [app] Ending request: urn:request:c1cb7771-5f4e-430d-8e09-8a678c366370 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c1cb7771-5f4e-430d-8e09-8a678c366370', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:55,073 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:14:55,073 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:55,074 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:55,074 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:55,074 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:55,079 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:55,079 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:55,085 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:55,087 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:55,087 [261] [DEBUG] [app] Ending request: urn:request:136a75ec-5e05-4796-b2f8-ee724db739d0 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:136a75ec-5e05-4796-b2f8-ee724db739d0', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:55,088 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:55,088 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.023) gunicorn-web stdout | 2025-11-04 09:14:55,122 [261] [DEBUG] [app] Starting request: urn:request:e1113401-505f-4321-9785-cda3bd625fe8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:14:55,125 [255] [DEBUG] [app] Starting request: urn:request:af7189b8-5a0b-4789-9da3-79e5b0d409e1 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:14:55,126 [255] [DEBUG] [app] Ending request: urn:request:af7189b8-5a0b-4789-9da3-79e5b0d409e1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:af7189b8-5a0b-4789-9da3-79e5b0d409e1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.003) gunicorn-registry stdout | 2025-11-04 09:14:55,126 [255] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:14:55,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:55,128 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:14:55,133 [261] [DEBUG] [app] Starting request: urn:request:76e0c89e-760e-411c-99d4-c4e3b8519238 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:14:55,133 [261] [DEBUG] [app] Ending request: urn:request:76e0c89e-760e-411c-99d4-c4e3b8519238 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:76e0c89e-760e-411c-99d4-c4e3b8519238', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:14:55,133 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:14:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.005 159 0.005) gunicorn-web stdout | 2025-11-04 09:14:55,134 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:14:55,134 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:14:55,135 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:14:55,135 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:14:55,139 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:14:55,139 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:14:55,144 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:14:55,145 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:14:55,146 [261] [DEBUG] [app] Ending request: urn:request:e1113401-505f-4321-9785-cda3bd625fe8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e1113401-505f-4321-9785-cda3bd625fe8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:14:55,146 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:14:55,147 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:14:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:14:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.025 118 0.025) gcworker stdout | 2025-11-04 09:14:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:14:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:07.663502+00:00 (in 12.001339 seconds) gcworker stdout | 2025-11-04 09:14:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:25 GMT)" (scheduled at 2025-11-04 09:14:55.661677+00:00) gcworker stdout | 2025-11-04 09:14:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038095662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:14:55,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:14:55,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:14:55,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:14:55,788 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:14:56,176 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:14:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:14:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:07.766253+00:00 (in 9.999523 seconds) notificationworker stdout | 2025-11-04 09:14:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:07 GMT)" (scheduled at 2025-11-04 09:14:57.766253+00:00) notificationworker stdout | 2025-11-04 09:14:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:14:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 57, 767132), True, datetime.datetime(2025, 11, 4, 9, 14, 57, 767132), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:14:57,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:14:57,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:14:57,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:14:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:14:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:09.676299+00:00 (in 9.999521 seconds) proxycacheblobworker stdout | 2025-11-04 09:14:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:09 GMT)" (scheduled at 2025-11-04 09:14:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:14:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:14:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 14, 59, 677133), True, datetime.datetime(2025, 11, 4, 9, 14, 59, 677133), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:14:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:14:59,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:14:59,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:14:59,982 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,087 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:52.159714+00:00 (in 52.072641 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:00 GMT)" (scheduled at 2025-11-04 09:15:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) globalpromstats stdout | 2025-11-04 09:15:00,090 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,095 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:15:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:00 GMT)" executed successfully builder stdout | 2025-11-04 09:15:00,385 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:15:01,194 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:15:01,372 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:15:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:19.706478+00:00 (in 17.998687 seconds) securityworker stdout | 2025-11-04 09:15:01,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:31 GMT)" (scheduled at 2025-11-04 09:15:01.707296+00:00) securityworker stdout | 2025-11-04 09:15:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:01,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:01,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:01,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:01,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:01,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:15:01,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:01,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:01,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:01,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 1, 713701), 1, 49]) securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:01,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:01,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 1, 713701), 1, 49]) securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:01,731 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:01,732 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:15:02,082 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:15:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:32.081579+00:00 (in 29.999297 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:15:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:32 GMT)" (scheduled at 2025-11-04 09:15:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:15:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:15:02,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:15:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:15:02,568 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:15:04,178 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:15:04,290 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:15:05,365 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:15:05,862 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:15:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:15:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:40.180734+00:00 (in 33.913921 seconds) repositorygcworker stdout | 2025-11-04 09:15:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:06 GMT)" (scheduled at 2025-11-04 09:15:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:15:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:15:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:15:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:15:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:33.568037+00:00 (in 26.998005 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:15:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:06 GMT)" (scheduled at 2025-11-04 09:15:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:15:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:15:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:15:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:15:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:37.077358+00:00 (in 29.999387 seconds) autopruneworker stdout | 2025-11-04 09:15:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:37 GMT)" (scheduled at 2025-11-04 09:15:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:15:07,085 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244107083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:15:07,088 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:15:07,088 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:15:07,089 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:15:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:25.661677+00:00 (in 17.997605 seconds) gcworker stdout | 2025-11-04 09:15:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:37 GMT)" (scheduled at 2025-11-04 09:15:07.663502+00:00) gcworker stdout | 2025-11-04 09:15:07,665 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:15:07,673 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247407672, None, 1, 0]) gcworker stdout | 2025-11-04 09:15:07,675 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:07,675 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:17.766253+00:00 (in 9.999479 seconds) notificationworker stdout | 2025-11-04 09:15:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:17 GMT)" (scheduled at 2025-11-04 09:15:07.766253+00:00) notificationworker stdout | 2025-11-04 09:15:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:07,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 7, 767222), True, datetime.datetime(2025, 11, 4, 9, 15, 7, 767222), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:15:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:15:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:07.770322+00:00 (in 59.999416 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:15:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:07 GMT)" (scheduled at 2025-11-04 09:15:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:15:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:15:07,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:07,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:07,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:15:07,778 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:15:07,778 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:15:08,373 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:15:09,490 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:19.676299+00:00 (in 9.999434 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:19 GMT)" (scheduled at 2025-11-04 09:15:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 9, 677250), True, datetime.datetime(2025, 11, 4, 9, 15, 9, 677250), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:09,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:09,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:09,687 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:15:09,752 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:10,066 [261] [DEBUG] [app] Starting request: urn:request:f50c0f72-a924-489e-b8bb-6910af8f8d1a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:15:10,067 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:15:10,070 [260] [DEBUG] [app] Starting request: urn:request:32a8d059-959a-4b7b-aa73-3e65f8e7d7be (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:10,071 [260] [DEBUG] [app] Ending request: urn:request:32a8d059-959a-4b7b-aa73-3e65f8e7d7be (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:32a8d059-959a-4b7b-aa73-3e65f8e7d7be', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:10,071 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:10,071 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:15:10,073 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:10,074 [261] [DEBUG] [app] Starting request: urn:request:6b325fb2-8e9b-4351-9286-baf529f97f5a (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:10,075 [261] [DEBUG] [app] Ending request: urn:request:6b325fb2-8e9b-4351-9286-baf529f97f5a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6b325fb2-8e9b-4351-9286-baf529f97f5a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:10,075 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:10,076 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:10,076 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:10,076 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:10,076 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:10,081 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:10,082 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:10,086 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:10,088 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:10,088 [261] [DEBUG] [app] Ending request: urn:request:f50c0f72-a924-489e-b8bb-6910af8f8d1a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:f50c0f72-a924-489e-b8bb-6910af8f8d1a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:10,089 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:10,089 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:15:10,122 [261] [DEBUG] [app] Starting request: urn:request:a5f0556a-d158-4946-bfa4-17240ac0bde5 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:15:10,124 [255] [DEBUG] [app] Starting request: urn:request:c24dc8fe-65c5-4f7c-aa15-46de6b9954d2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:10,125 [255] [DEBUG] [app] Ending request: urn:request:c24dc8fe-65c5-4f7c-aa15-46de6b9954d2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c24dc8fe-65c5-4f7c-aa15-46de6b9954d2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:10,125 [255] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:10,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:15:10,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:10,129 [254] [DEBUG] [app] Starting request: urn:request:01589d4f-4dd2-4d71-8574-ae91e60e5660 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:10,129 [254] [DEBUG] [app] Ending request: urn:request:01589d4f-4dd2-4d71-8574-ae91e60e5660 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:01589d4f-4dd2-4d71-8574-ae91e60e5660', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:10,130 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:10,130 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:10,130 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:10,130 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:10,130 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:10,135 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:10,135 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:10,140 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:10,141 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:10,142 [261] [DEBUG] [app] Ending request: urn:request:a5f0556a-d158-4946-bfa4-17240ac0bde5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a5f0556a-d158-4946-bfa4-17240ac0bde5', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:10,142 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:10,143 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) securityworker stdout | 2025-11-04 09:15:10,175 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:15:10,184 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:15:10,982 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:15:11,278 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:15:11,568 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:12,244 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:12,253 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:12,253 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:15:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:15:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:38.480592+00:00 (in 25.997892 seconds) exportactionlogsworker stdout | 2025-11-04 09:15:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:12 GMT)" (scheduled at 2025-11-04 09:15:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:15:12,483 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:15:12,483 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:15:12,687 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:13,071 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:15:13,474 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:14,287 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:14,745 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:14,760 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:14,766 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:14,777 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:14,919 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:15:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:44.980431+00:00 (in 29.999517 seconds) buildlogsarchiver stdout | 2025-11-04 09:15:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:44 GMT)" (scheduled at 2025-11-04 09:15:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:15:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 15, 14, 981184), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:15:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:15:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:15:16,242 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,328 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,370 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,390 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,397 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,410 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,415 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,423 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:16,423 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:27.766253+00:00 (in 9.999477 seconds) notificationworker stdout | 2025-11-04 09:15:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:27 GMT)" (scheduled at 2025-11-04 09:15:17.766253+00:00) notificationworker stdout | 2025-11-04 09:15:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 17, 767111), True, datetime.datetime(2025, 11, 4, 9, 15, 17, 767111), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:17,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:17,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:15:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:29.676299+00:00 (in 9.999481 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:29 GMT)" (scheduled at 2025-11-04 09:15:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 19, 677104), True, datetime.datetime(2025, 11, 4, 9, 15, 19, 677104), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:15:19,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:19,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:31.707296+00:00 (in 12.000032 seconds) securityworker stdout | 2025-11-04 09:15:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:49 GMT)" (scheduled at 2025-11-04 09:15:19.706478+00:00) securityworker stdout | 2025-11-04 09:15:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:19,712 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:19,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:15:19,720 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:19,720 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:15:21,104 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:25,065 [258] [DEBUG] [app] Starting request: urn:request:6d0ac63a-48d3-4158-989b-635fedeed041 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:15:25,066 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:15:25,069 [259] [DEBUG] [app] Starting request: urn:request:a616940e-ca2f-4cd3-aae1-1dd56f82d86c (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:25,069 [259] [DEBUG] [app] Ending request: urn:request:a616940e-ca2f-4cd3-aae1-1dd56f82d86c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a616940e-ca2f-4cd3-aae1-1dd56f82d86c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:25,069 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:15:25,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:25,071 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:25,086 [258] [DEBUG] [app] Starting request: urn:request:90d62817-dd19-45b7-b1eb-a79de3c4964b (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:25,086 [258] [DEBUG] [app] Ending request: urn:request:90d62817-dd19-45b7-b1eb-a79de3c4964b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:90d62817-dd19-45b7-b1eb-a79de3c4964b', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:25,087 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:25,087 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:25,087 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:25,088 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:25,088 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:25,093 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:25,093 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:25,099 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:25,100 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:25,101 [258] [DEBUG] [app] Ending request: urn:request:6d0ac63a-48d3-4158-989b-635fedeed041 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6d0ac63a-48d3-4158-989b-635fedeed041', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:25,101 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:25,102 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.038 118 0.037) gunicorn-web stdout | 2025-11-04 09:15:25,122 [258] [DEBUG] [app] Starting request: urn:request:683d7493-5755-4d5e-8b12-2a0e56ca88a8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:15:25,124 [260] [DEBUG] [app] Starting request: urn:request:6a9178fc-2596-4680-8f0c-70e34d8f16a0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:25,125 [260] [DEBUG] [app] Ending request: urn:request:6a9178fc-2596-4680-8f0c-70e34d8f16a0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6a9178fc-2596-4680-8f0c-70e34d8f16a0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:25,126 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:15:25,126 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:25,127 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:25,128 [254] [DEBUG] [app] Starting request: urn:request:555d8913-fd1f-4ece-8384-46b621cc5f1f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:25,129 [254] [DEBUG] [app] Ending request: urn:request:555d8913-fd1f-4ece-8384-46b621cc5f1f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:555d8913-fd1f-4ece-8384-46b621cc5f1f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:25,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:25,129 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:25,130 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:25,130 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:25,130 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:25,134 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:25,134 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:25,139 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:25,141 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:25,142 [258] [DEBUG] [app] Ending request: urn:request:683d7493-5755-4d5e-8b12-2a0e56ca88a8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:683d7493-5755-4d5e-8b12-2a0e56ca88a8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:25,142 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:25,143 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.022) gcworker stdout | 2025-11-04 09:15:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:37.663502+00:00 (in 12.001295 seconds) gcworker stdout | 2025-11-04 09:15:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:55 GMT)" (scheduled at 2025-11-04 09:15:25.661677+00:00) gcworker stdout | 2025-11-04 09:15:25,662 [66] [DEBUG] [peewee] ('SELECT DISTINCT "t1"."removed_tag_expiration_s" FROM "user" AS "t1" LIMIT %s', [100]) gcworker stdout | 2025-11-04 09:15:25,671 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038125670, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:15:25,674 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:15:25,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:25,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:15:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:25,806 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:26,193 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:27.768179+00:00 (in 0.001424 seconds) notificationworker stdout | 2025-11-04 09:15:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:37 GMT)" (scheduled at 2025-11-04 09:15:27.766253+00:00) notificationworker stdout | 2025-11-04 09:15:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 27, 767181), True, datetime.datetime(2025, 11, 4, 9, 15, 27, 767181), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:37.766253+00:00 (in 9.997693 seconds) notificationworker stdout | 2025-11-04 09:15:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:27 GMT)" (scheduled at 2025-11-04 09:15:27.768179+00:00) notificationworker stdout | 2025-11-04 09:15:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:15:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:39.676299+00:00 (in 9.999427 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:29,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:39 GMT)" (scheduled at 2025-11-04 09:15:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 29, 677282), True, datetime.datetime(2025, 11, 4, 9, 15, 29, 677282), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:15:30,001 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:15:30,107 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:15:30,401 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:15:31,213 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:15:31,390 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:15:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:49.706478+00:00 (in 17.998730 seconds) securityworker stdout | 2025-11-04 09:15:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:01 GMT)" (scheduled at 2025-11-04 09:15:31.707296+00:00) securityworker stdout | 2025-11-04 09:15:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:31,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:31,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:31,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 31, 713384), 1, 49]) securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:31,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:31,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 31, 713384), 1, 49]) securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:15:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:31,729 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:31,729 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:15:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:15:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:02.081579+00:00 (in 29.999487 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:15:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:02 GMT)" (scheduled at 2025-11-04 09:15:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:15:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:15:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:15:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:15:32,585 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:15:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:53.560131+00:00 (in 19.997746 seconds) namespacegcworker stdout | 2025-11-04 09:15:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" (scheduled at 2025-11-04 09:15:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:15:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:15:33,563 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:15:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:15:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:06.569529+00:00 (in 33.001015 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:15:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" (scheduled at 2025-11-04 09:15:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:15:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:15:33,570 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 33, 569072), True, datetime.datetime(2025, 11, 4, 9, 15, 33, 569072), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:15:33,579 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:15:33,579 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:15:33,579 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:15:34,212 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:15:34,306 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:15:35,383 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:15:35,897 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:15:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:15:37,078 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:07.077358+00:00 (in 29.999368 seconds) autopruneworker stdout | 2025-11-04 09:15:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:07 GMT)" (scheduled at 2025-11-04 09:15:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:15:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244137083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:15:37,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:15:37,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:15:37,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:15:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:55.661677+00:00 (in 17.997650 seconds) gcworker stdout | 2025-11-04 09:15:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:07 GMT)" (scheduled at 2025-11-04 09:15:37.663502+00:00) gcworker stdout | 2025-11-04 09:15:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:15:37,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247437671, None, 1, 0]) gcworker stdout | 2025-11-04 09:15:37,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:37,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:47.766253+00:00 (in 9.999573 seconds) notificationworker stdout | 2025-11-04 09:15:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:47 GMT)" (scheduled at 2025-11-04 09:15:37.766253+00:00) notificationworker stdout | 2025-11-04 09:15:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 37, 766981), True, datetime.datetime(2025, 11, 4, 9, 15, 37, 766981), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:15:38,392 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:15:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:15:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:55.481710+00:00 (in 17.000599 seconds) exportactionlogsworker stdout | 2025-11-04 09:15:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:38 GMT)" (scheduled at 2025-11-04 09:15:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:15:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:15:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 38, 481517), True, datetime.datetime(2025, 11, 4, 9, 15, 38, 481517), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:15:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:15:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:15:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:15:39,507 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:47.679968+00:00 (in 8.003088 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:39,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:49 GMT)" (scheduled at 2025-11-04 09:15:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 39, 677266), True, datetime.datetime(2025, 11, 4, 9, 15, 39, 677266), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:39,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:39,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:39,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:15:39,767 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:40,066 [254] [DEBUG] [app] Starting request: urn:request:8925108a-679f-4c86-a272-eb5d9c6d7924 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:15:40,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:15:40,070 [250] [DEBUG] [app] Starting request: urn:request:d39ecea4-1133-41e7-a43c-b1ded7bdf2db (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:40,070 [250] [DEBUG] [app] Ending request: urn:request:d39ecea4-1133-41e7-a43c-b1ded7bdf2db (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d39ecea4-1133-41e7-a43c-b1ded7bdf2db', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:40,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:15:40,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:40,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:40,073 [254] [DEBUG] [app] Starting request: urn:request:d2ec4fbc-f155-4b3d-a605-aff6e1a066dc (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:40,074 [254] [DEBUG] [app] Ending request: urn:request:d2ec4fbc-f155-4b3d-a605-aff6e1a066dc (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d2ec4fbc-f155-4b3d-a605-aff6e1a066dc', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:15:40,074 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:40,075 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:40,075 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:40,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:40,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:40,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:40,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:40,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:40,089 [254] [DEBUG] [app] Ending request: urn:request:8925108a-679f-4c86-a272-eb5d9c6d7924 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8925108a-679f-4c86-a272-eb5d9c6d7924', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:40,089 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:15:40,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:15:40,124 [261] [DEBUG] [app] Starting request: urn:request:99ff42a4-a9a1-4261-9461-55970bde0f83 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:15:40,125 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:15:40,127 [250] [DEBUG] [app] Starting request: urn:request:e1e98ad9-38ed-425a-a8fe-0db6b498fde2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:40,128 [250] [DEBUG] [app] Ending request: urn:request:e1e98ad9-38ed-425a-a8fe-0db6b498fde2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e1e98ad9-38ed-425a-a8fe-0db6b498fde2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:40,128 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.003) gunicorn-web stdout | 2025-11-04 09:15:40,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:40,134 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:40,135 [254] [DEBUG] [app] Starting request: urn:request:d5de68b1-c6c8-4fca-a44c-e1eb3fd05120 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:40,136 [254] [DEBUG] [app] Ending request: urn:request:d5de68b1-c6c8-4fca-a44c-e1eb3fd05120 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d5de68b1-c6c8-4fca-a44c-e1eb3fd05120', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:40,136 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:40,136 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:40,137 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:40,137 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:40,137 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:40,142 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:40,142 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:40,146 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:40,147 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:40,148 [261] [DEBUG] [app] Ending request: urn:request:99ff42a4-a9a1-4261-9461-55970bde0f83 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:99ff42a4-a9a1-4261-9461-55970bde0f83', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:40,148 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:40,149 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.026) repositorygcworker stdout | 2025-11-04 09:15:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:15:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:06.265965+00:00 (in 26.084710 seconds) repositorygcworker stdout | 2025-11-04 09:15:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:40 GMT)" (scheduled at 2025-11-04 09:15:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:15:40,182 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:15:40,183 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 40, 182133), True, datetime.datetime(2025, 11, 4, 9, 15, 40, 182133), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:15:40,191 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:15:40,191 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:15:40,191 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:15:40,194 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:15:40,218 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:15:40,999 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:15:41,294 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:15:41,585 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:42,263 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:42,276 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:15:42,278 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:15:42,705 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:43,090 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:15:43,491 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:15:44,304 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:44,765 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:44,781 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:44,788 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:44,796 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:15:44,939 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:15:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:14.980431+00:00 (in 29.999490 seconds) buildlogsarchiver stdout | 2025-11-04 09:15:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:14 GMT)" (scheduled at 2025-11-04 09:15:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:15:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 15, 44, 981199), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:15:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:15:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:15:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:15:46,261 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,347 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,389 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,410 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,418 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,433 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,433 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,442 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:15:46,444 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:49.676299+00:00 (in 1.995366 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:47,681 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:47 GMT)" (scheduled at 2025-11-04 09:15:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:15:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:57.766253+00:00 (in 9.999574 seconds) notificationworker stdout | 2025-11-04 09:15:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:57 GMT)" (scheduled at 2025-11-04 09:15:47.766253+00:00) notificationworker stdout | 2025-11-04 09:15:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 47, 766986), True, datetime.datetime(2025, 11, 4, 9, 15, 47, 766986), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:47,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:47,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:47,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:15:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:15:59.676299+00:00 (in 9.999449 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:59 GMT)" (scheduled at 2025-11-04 09:15:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 49, 677235), True, datetime.datetime(2025, 11, 4, 9, 15, 49, 677235), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:49,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:49,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:49,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:15:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:15:49,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:15:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:01.707296+00:00 (in 12.000047 seconds) securityworker stdout | 2025-11-04 09:15:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:19 GMT)" (scheduled at 2025-11-04 09:15:49.706478+00:00) securityworker stdout | 2025-11-04 09:15:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:15:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:15:49,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:15:49,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:15:49,717 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:15:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 29, 39]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,721 [92] [DEBUG] [util.migrate.allocator] Left range 29-39 securityworker stdout | 2025-11-04 09:15:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stdout | 2025-11-04 09:15:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 39, 49]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:15:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 29-39 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Already merged with block 29-39 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-29 securityworker stdout | 2025-11-04 09:15:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stdout | 2025-11-04 09:15:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 8, 18]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 8-18 by worker securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 8-18 by worker securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 8-18 securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 8-18 securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stdout | 2025-11-04 09:15:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,727 [92] [DEBUG] [util.migrate.allocator] Right range 8-18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 8-18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 8-18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 8-18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-8 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-8 securityworker stdout | 2025-11-04 09:15:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Merging with block 8-18 securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 18 securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 18-29 securityworker stdout | 2025-11-04 09:15:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-29 securityworker stdout | 2025-11-04 09:15:49,730 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 8-18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 18 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 18-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stdout | 2025-11-04 09:15:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 18, 28]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 28-29 securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-29 securityworker stdout | 2025-11-04 09:15:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 28 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 28-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:15:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 28, 38]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:15:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:15:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 28 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 38 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 28 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 38 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 38-28 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 38-28 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 9, 19]) securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-49 securityworker stdout | 2025-11-04 09:15:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 31, 41]) securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-31 securityworker stdout | 2025-11-04 09:15:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-31 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:15:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 19, 29]) securityworker stdout | 2025-11-04 09:15:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:15:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:15:49,740 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-19 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-29 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Right range 9-29 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stdout | 2025-11-04 09:15:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 1, 11]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:15:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-29 securityworker stdout | 2025-11-04 09:15:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 29-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 29-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Left range 31-41 securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:15:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 41, 49]) securityworker stdout | 2025-11-04 09:15:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:15:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:15:49,744 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 31-41 securityworker stdout | 2025-11-04 09:15:49,744 [92] [DEBUG] [util.migrate.allocator] Already merged with block 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 29-31 securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-31 securityworker stdout | 2025-11-04 09:15:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 29-31 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-31 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stdout | 2025-11-04 09:15:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 29, 39]) securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 39-29 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:15:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 39-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 9, 19]) securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,750 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stdout | 2025-11-04 09:15:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:15:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 31, 41]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:15:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:15:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:15:49,752 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-31 securityworker stdout | 2025-11-04 09:15:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-31 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:15:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 19, 29]) securityworker stdout | 2025-11-04 09:15:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-19 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-19 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-29 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Left range 9-29 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-31 securityworker stdout | 2025-11-04 09:15:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Left range 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-31 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stdout | 2025-11-04 09:15:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 29, 39]) securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-29 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 9-29 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-41 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 9-41 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Right range 9-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:15:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 1, 11]) securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:15:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:15:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-41 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Total range: 41-49 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:15:49,760 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 41-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:15:49,761 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 10, 49, 712165), 41, 49]) securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 41 securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] Total range: 49-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-41 securityworker stderr | 2025-11-04 09:15:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:15:49,762 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:15:49,762 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:15:51,121 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:00.086435+00:00 (in 7.926168 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:52 GMT)" (scheduled at 2025-11-04 09:15:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:15:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:15:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:15:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:33.561829+00:00 (in 40.001132 seconds) namespacegcworker stdout | 2025-11-04 09:15:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:53 GMT)" (scheduled at 2025-11-04 09:15:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:15:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:15:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 53, 561019), True, datetime.datetime(2025, 11, 4, 9, 15, 53, 561019), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:15:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:15:53,570 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:15:53,570 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:16:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:15:55,066 [254] [DEBUG] [app] Starting request: urn:request:66d22fc9-a162-4a93-a7af-d3e88cc26447 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:15:55,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:15:55,069 [250] [DEBUG] [app] Starting request: urn:request:2135c071-249d-4c3e-87f1-417a3bfad4e9 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:55,070 [250] [DEBUG] [app] Ending request: urn:request:2135c071-249d-4c3e-87f1-417a3bfad4e9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:2135c071-249d-4c3e-87f1-417a3bfad4e9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:15:55,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:15:55,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:55,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:55,073 [261] [DEBUG] [app] Starting request: urn:request:b78a7905-dac4-4f82-9ab0-d9c2f5e88f7a (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:55,074 [261] [DEBUG] [app] Ending request: urn:request:b78a7905-dac4-4f82-9ab0-d9c2f5e88f7a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b78a7905-dac4-4f82-9ab0-d9c2f5e88f7a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:55,074 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:55,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:55,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:55,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:55,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:55,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:55,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:55,087 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:55,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:55,089 [254] [DEBUG] [app] Ending request: urn:request:66d22fc9-a162-4a93-a7af-d3e88cc26447 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:66d22fc9-a162-4a93-a7af-d3e88cc26447', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:55,089 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.025 118 0.025) gunicorn-web stdout | 2025-11-04 09:15:55,090 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:15:55,122 [254] [DEBUG] [app] Starting request: urn:request:88fe82aa-463c-45bf-bf44-8d07b2c18ba8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:15:55,129 [250] [DEBUG] [app] Starting request: urn:request:be8b32ba-7819-4b5a-86cc-6b7740f583e9 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:15:55,129 [250] [DEBUG] [app] Ending request: urn:request:be8b32ba-7819-4b5a-86cc-6b7740f583e9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:be8b32ba-7819-4b5a-86cc-6b7740f583e9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 162 0.002) gunicorn-registry stdout | 2025-11-04 09:15:55,130 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:15:55,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:55,132 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:15:55,133 [261] [DEBUG] [app] Starting request: urn:request:89babb84-fd47-4caf-8da7-32d8ebf6103d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:15:55,134 [261] [DEBUG] [app] Ending request: urn:request:89babb84-fd47-4caf-8da7-32d8ebf6103d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:89babb84-fd47-4caf-8da7-32d8ebf6103d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:15:55,134 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:15:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:15:55,134 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:15:55,135 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:15:55,135 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:15:55,135 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:15:55,139 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:15:55,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:15:55,144 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:15:55,146 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:15:55,147 [254] [DEBUG] [app] Ending request: urn:request:88fe82aa-463c-45bf-bf44-8d07b2c18ba8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:88fe82aa-463c-45bf-bf44-8d07b2c18ba8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:15:55,148 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:15:55,148 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:15:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:15:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.027 118 0.026) exportactionlogsworker stdout | 2025-11-04 09:15:55,481 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:15:55,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:12.482065+00:00 (in 16.999874 seconds) exportactionlogsworker stdout | 2025-11-04 09:15:55,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:20:55 GMT)" (scheduled at 2025-11-04 09:15:55.481710+00:00) exportactionlogsworker stdout | 2025-11-04 09:15:55,483 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:15:55,490 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), True, datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:15:55,491 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), True, datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 15, 55, 482583), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:15:55,493 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:15:55,493 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:20:55 GMT)" executed successfully gcworker stdout | 2025-11-04 09:15:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:15:55,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:07.663502+00:00 (in 12.000404 seconds) gcworker stdout | 2025-11-04 09:15:55,663 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:25 GMT)" (scheduled at 2025-11-04 09:15:55.661677+00:00) gcworker stdout | 2025-11-04 09:15:55,664 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038155663, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:15:55,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:15:55,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:15:55,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:15:55,821 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:15:56,210 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:15:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:15:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:07.766253+00:00 (in 9.999499 seconds) notificationworker stdout | 2025-11-04 09:15:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:07 GMT)" (scheduled at 2025-11-04 09:15:57.766253+00:00) notificationworker stdout | 2025-11-04 09:15:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:15:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 57, 767124), True, datetime.datetime(2025, 11, 4, 9, 15, 57, 767124), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:15:57,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:15:57,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:15:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:15:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:15:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:09.676299+00:00 (in 9.999450 seconds) proxycacheblobworker stdout | 2025-11-04 09:15:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:09 GMT)" (scheduled at 2025-11-04 09:15:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:15:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:15:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 15, 59, 677177), True, datetime.datetime(2025, 11, 4, 9, 15, 59, 677177), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:15:59,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:15:59,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:15:59,687 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:16:00,019 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,087 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:52.159714+00:00 (in 52.072685 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:00 GMT)" (scheduled at 2025-11-04 09:16:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,096 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:16:00,096 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:00 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:16:00,123 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:00,418 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:16:01,233 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:16:01,408 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:16:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:19.706478+00:00 (in 17.998635 seconds) securityworker stdout | 2025-11-04 09:16:01,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:31 GMT)" (scheduled at 2025-11-04 09:16:01.707296+00:00) securityworker stdout | 2025-11-04 09:16:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:01,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:01,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:01,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:01,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:01,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 1, 713300), 1, 49]) securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:01,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:01,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 1, 713300), 1, 49]) securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:01,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:01,729 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:01,729 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:16:02,082 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:16:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:32.081579+00:00 (in 29.999245 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:16:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:32 GMT)" (scheduled at 2025-11-04 09:16:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:16:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:16:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:16:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:16:02,612 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:16:04,229 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:16:04,323 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:16:05,402 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:16:05,913 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:16:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:16:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:40.180734+00:00 (in 33.914258 seconds) repositorygcworker stdout | 2025-11-04 09:16:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:06 GMT)" (scheduled at 2025-11-04 09:16:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:16:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:16:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:16:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:16:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:33.568037+00:00 (in 26.997848 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:16:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:06 GMT)" (scheduled at 2025-11-04 09:16:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:16:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:16:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:16:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:16:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:37.077358+00:00 (in 29.999498 seconds) autopruneworker stdout | 2025-11-04 09:16:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:37 GMT)" (scheduled at 2025-11-04 09:16:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:16:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244167082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:16:07,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:16:07,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:16:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:16:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:25.661677+00:00 (in 17.997669 seconds) gcworker stdout | 2025-11-04 09:16:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:37 GMT)" (scheduled at 2025-11-04 09:16:07.663502+00:00) gcworker stdout | 2025-11-04 09:16:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:16:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247467670, None, 1, 0]) gcworker stdout | 2025-11-04 09:16:07,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:07,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:17.766253+00:00 (in 9.999495 seconds) notificationworker stdout | 2025-11-04 09:16:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:17 GMT)" (scheduled at 2025-11-04 09:16:07.766253+00:00) notificationworker stdout | 2025-11-04 09:16:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 7, 767060), True, datetime.datetime(2025, 11, 4, 9, 16, 7, 767060), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:16:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:16:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:07.770322+00:00 (in 59.999496 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:16:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:07 GMT)" (scheduled at 2025-11-04 09:16:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:16:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:16:07,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:16:07,777 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:16:07,777 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:16:08,410 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:16:09,523 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:19.676299+00:00 (in 9.999451 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:19 GMT)" (scheduled at 2025-11-04 09:16:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 9, 677240), True, datetime.datetime(2025, 11, 4, 9, 16, 9, 677240), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:16:09,785 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:10,066 [258] [DEBUG] [app] Starting request: urn:request:d1f8ff4f-bf40-4fcc-8b1c-5d57e65ccd88 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:16:10,068 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:16:10,070 [250] [DEBUG] [app] Starting request: urn:request:183956b0-7863-47c0-84e9-9c03a79cec46 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:10,070 [250] [DEBUG] [app] Ending request: urn:request:183956b0-7863-47c0-84e9-9c03a79cec46 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:183956b0-7863-47c0-84e9-9c03a79cec46', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:10,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:10,071 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:10,072 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:10,074 [258] [DEBUG] [app] Starting request: urn:request:3017cc51-abdf-48c7-ab7d-b7074653a886 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:10,074 [258] [DEBUG] [app] Ending request: urn:request:3017cc51-abdf-48c7-ab7d-b7074653a886 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3017cc51-abdf-48c7-ab7d-b7074653a886', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:10,074 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:10,075 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:10,075 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:10,075 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:10,075 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:10,081 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:10,082 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:10,086 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:10,088 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:10,089 [258] [DEBUG] [app] Ending request: urn:request:d1f8ff4f-bf40-4fcc-8b1c-5d57e65ccd88 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d1f8ff4f-bf40-4fcc-8b1c-5d57e65ccd88', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:10,089 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:10,089 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.025) gunicorn-web stdout | 2025-11-04 09:16:10,123 [258] [DEBUG] [app] Starting request: urn:request:6f12c428-d171-4a64-beb0-f76a379ea09f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:16:10,125 [250] [DEBUG] [app] Starting request: urn:request:3ff5b2fe-4433-4efe-992d-67464dc3ebdb (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:10,126 [250] [DEBUG] [app] Ending request: urn:request:3ff5b2fe-4433-4efe-992d-67464dc3ebdb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3ff5b2fe-4433-4efe-992d-67464dc3ebdb', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:16:10,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:10,126 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:10,127 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:10,130 [254] [DEBUG] [app] Starting request: urn:request:53f40286-6a92-4d08-9d7d-e343c96ae070 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:10,130 [254] [DEBUG] [app] Ending request: urn:request:53f40286-6a92-4d08-9d7d-e343c96ae070 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:53f40286-6a92-4d08-9d7d-e343c96ae070', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:10,131 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:16:10,131 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:10,132 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:10,132 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:10,132 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:10,137 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:10,137 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:10,148 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:10,150 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:10,151 [258] [DEBUG] [app] Ending request: urn:request:6f12c428-d171-4a64-beb0-f76a379ea09f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:6f12c428-d171-4a64-beb0-f76a379ea09f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:10,151 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:10,152 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.029 118 0.029) securityworker stdout | 2025-11-04 09:16:10,210 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:16:10,235 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:16:11,015 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:16:11,310 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:16:11,604 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:12,283 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:12,295 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:12,297 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:16:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:16:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:38.480592+00:00 (in 25.997966 seconds) exportactionlogsworker stdout | 2025-11-04 09:16:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:12 GMT)" (scheduled at 2025-11-04 09:16:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:16:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:16:12,483 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:16:12,721 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:13,121 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:13,506 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:14,322 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:14,785 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:14,801 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:14,810 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:14,814 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:14,959 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:16:14,981 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:44.980431+00:00 (in 29.999431 seconds) buildlogsarchiver stdout | 2025-11-04 09:16:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:44 GMT)" (scheduled at 2025-11-04 09:16:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:16:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 16, 14, 981317), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:14,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:16:14,991 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:16:14,991 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:44 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:16:16,281 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,367 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,408 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,430 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,438 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,451 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,456 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,460 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:16,465 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:27.766253+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 09:16:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:27 GMT)" (scheduled at 2025-11-04 09:16:17.766253+00:00) notificationworker stdout | 2025-11-04 09:16:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 17, 767128), True, datetime.datetime(2025, 11, 4, 9, 16, 17, 767128), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:17,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:17,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:16:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:29.676299+00:00 (in 9.999422 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:19,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:29 GMT)" (scheduled at 2025-11-04 09:16:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 19, 677269), True, datetime.datetime(2025, 11, 4, 9, 16, 19, 677269), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:19,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:19,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:19,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:16:19,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:19,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:31.707296+00:00 (in 11.999973 seconds) securityworker stdout | 2025-11-04 09:16:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:49 GMT)" (scheduled at 2025-11-04 09:16:19.706478+00:00) securityworker stdout | 2025-11-04 09:16:19,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:19,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:19,712 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:19,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:16:19,719 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:19,719 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:16:21,137 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:25,067 [254] [DEBUG] [app] Starting request: urn:request:ee0bc9da-dbdc-4a06-a673-11b9e8090f35 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:16:25,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:16:25,071 [250] [DEBUG] [app] Starting request: urn:request:4c7825c6-3832-409e-9c41-f5839e94f08f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:25,072 [250] [DEBUG] [app] Ending request: urn:request:4c7825c6-3832-409e-9c41-f5839e94f08f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4c7825c6-3832-409e-9c41-f5839e94f08f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:16:25,073 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:25,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:25,074 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:25,075 [254] [DEBUG] [app] Starting request: urn:request:d3348379-56f3-4bb2-959b-b8079c0b1f2c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:25,076 [254] [DEBUG] [app] Ending request: urn:request:d3348379-56f3-4bb2-959b-b8079c0b1f2c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d3348379-56f3-4bb2-959b-b8079c0b1f2c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:25,076 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:25,076 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:25,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:25,077 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:25,077 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:25,082 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:25,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:25,087 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:25,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:25,089 [254] [DEBUG] [app] Ending request: urn:request:ee0bc9da-dbdc-4a06-a673-11b9e8090f35 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ee0bc9da-dbdc-4a06-a673-11b9e8090f35', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:25,089 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:25,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) gunicorn-web stdout | 2025-11-04 09:16:25,122 [254] [DEBUG] [app] Starting request: urn:request:0ab7ecf8-ef54-4eef-8da8-e4a91f6e7517 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:16:25,124 [250] [DEBUG] [app] Starting request: urn:request:74e3d625-44f8-472c-90f6-42dc7f1f80aa (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:25,124 [250] [DEBUG] [app] Ending request: urn:request:74e3d625-44f8-472c-90f6-42dc7f1f80aa (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:74e3d625-44f8-472c-90f6-42dc7f1f80aa', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:25,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:25,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:25,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:25,128 [261] [DEBUG] [app] Starting request: urn:request:f602460b-c1ac-48c6-ad6d-2b14e3290446 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:25,129 [261] [DEBUG] [app] Ending request: urn:request:f602460b-c1ac-48c6-ad6d-2b14e3290446 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f602460b-c1ac-48c6-ad6d-2b14e3290446', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:16:25,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:25,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:25,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:25,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:25,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:25,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:25,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:25,141 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:25,142 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:25,143 [254] [DEBUG] [app] Ending request: urn:request:0ab7ecf8-ef54-4eef-8da8-e4a91f6e7517 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0ab7ecf8-ef54-4eef-8da8-e4a91f6e7517', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:25,143 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:25,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) gcworker stdout | 2025-11-04 09:16:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:37.663502+00:00 (in 12.001278 seconds) gcworker stdout | 2025-11-04 09:16:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:55 GMT)" (scheduled at 2025-11-04 09:16:25.661677+00:00) gcworker stdout | 2025-11-04 09:16:25,664 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038185663, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:16:25,674 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:16:25,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:25,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:16:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:25,839 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:26,226 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:27.768179+00:00 (in 0.001420 seconds) notificationworker stdout | 2025-11-04 09:16:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:37 GMT)" (scheduled at 2025-11-04 09:16:27.766253+00:00) notificationworker stdout | 2025-11-04 09:16:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 27, 767086), True, datetime.datetime(2025, 11, 4, 9, 16, 27, 767086), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:37.766253+00:00 (in 9.997605 seconds) notificationworker stdout | 2025-11-04 09:16:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:27 GMT)" (scheduled at 2025-11-04 09:16:27.768179+00:00) notificationworker stdout | 2025-11-04 09:16:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:16:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:39.676299+00:00 (in 9.999480 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:39 GMT)" (scheduled at 2025-11-04 09:16:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 29, 677166), True, datetime.datetime(2025, 11, 4, 9, 16, 29, 677166), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:29,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:29,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:16:30,052 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:16:30,140 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:16:30,435 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:16:31,254 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:16:31,426 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:16:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:31,708 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:49.706478+00:00 (in 17.998283 seconds) securityworker stdout | 2025-11-04 09:16:31,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:01 GMT)" (scheduled at 2025-11-04 09:16:31.707296+00:00) securityworker stdout | 2025-11-04 09:16:31,709 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:31,709 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:31,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:31,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:31,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:31,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:31,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 31, 713638), 1, 49]) securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:31,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:31,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 31, 713638), 1, 49]) securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:31,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:16:31,730 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:31,730 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:31,730 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:01 GMT)" executed successfully securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:16:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker pullstatsredisflushworker stdout | 2025-11-04 09:16:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:16:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:02.081579+00:00 (in 29.999353 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:16:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:02 GMT)" (scheduled at 2025-11-04 09:16:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:16:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:16:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:16:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:16:32,636 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:16:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:53.560131+00:00 (in 19.997816 seconds) namespacegcworker stdout | 2025-11-04 09:16:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" (scheduled at 2025-11-04 09:16:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:16:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:16:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:16:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:16:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:06.569529+00:00 (in 33.000903 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:16:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" (scheduled at 2025-11-04 09:16:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:16:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:16:33,570 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 33, 569031), True, datetime.datetime(2025, 11, 4, 9, 16, 33, 569031), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:16:33,578 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:16:33,578 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:16:33,578 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:16:34,245 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:16:34,339 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:16:35,435 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:16:35,930 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:16:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:16:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:07.077358+00:00 (in 29.999405 seconds) autopruneworker stdout | 2025-11-04 09:16:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:07 GMT)" (scheduled at 2025-11-04 09:16:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:16:37,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244197083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:16:37,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:16:37,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:16:37,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:16:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:55.661677+00:00 (in 17.997675 seconds) gcworker stdout | 2025-11-04 09:16:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:07 GMT)" (scheduled at 2025-11-04 09:16:37.663502+00:00) gcworker stdout | 2025-11-04 09:16:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:16:37,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247497671, None, 1, 0]) gcworker stdout | 2025-11-04 09:16:37,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:37,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:47.766253+00:00 (in 9.999451 seconds) notificationworker stdout | 2025-11-04 09:16:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:47 GMT)" (scheduled at 2025-11-04 09:16:37.766253+00:00) notificationworker stdout | 2025-11-04 09:16:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 37, 767138), True, datetime.datetime(2025, 11, 4, 9, 16, 37, 767138), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:16:38,427 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:16:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:16:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:12.482065+00:00 (in 34.000945 seconds) exportactionlogsworker stdout | 2025-11-04 09:16:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:38 GMT)" (scheduled at 2025-11-04 09:16:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:16:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:16:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 38, 481459), True, datetime.datetime(2025, 11, 4, 9, 16, 38, 481459), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:16:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:16:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:16:38,489 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:16:39,539 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:47.679968+00:00 (in 8.003239 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:49 GMT)" (scheduled at 2025-11-04 09:16:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 39, 677145), True, datetime.datetime(2025, 11, 4, 9, 16, 39, 677145), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:39,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:39,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:39,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:16:39,802 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:40,065 [254] [DEBUG] [app] Starting request: urn:request:7789c10c-0242-4fe3-9253-0f80d7b9a2bc (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:16:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:16:40,071 [259] [DEBUG] [app] Starting request: urn:request:e217afda-2d33-44c6-b262-7919f2df7782 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:40,071 [259] [DEBUG] [app] Ending request: urn:request:e217afda-2d33-44c6-b262-7919f2df7782 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e217afda-2d33-44c6-b262-7919f2df7782', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:16:40,072 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:40,072 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:40,074 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:40,076 [258] [DEBUG] [app] Starting request: urn:request:3140749e-a61e-4879-9198-f18c54d153c9 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:40,076 [258] [DEBUG] [app] Ending request: urn:request:3140749e-a61e-4879-9198-f18c54d153c9 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3140749e-a61e-4879-9198-f18c54d153c9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:16:40,076 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:40,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:40,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:40,077 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:40,077 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:40,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:40,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:40,087 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:40,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:40,089 [254] [DEBUG] [app] Ending request: urn:request:7789c10c-0242-4fe3-9253-0f80d7b9a2bc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7789c10c-0242-4fe3-9253-0f80d7b9a2bc', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:40,089 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:40,090 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.025) gunicorn-web stdout | 2025-11-04 09:16:40,123 [254] [DEBUG] [app] Starting request: urn:request:d575c906-e837-4f5f-b026-72ba08a6a197 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:16:40,125 [250] [DEBUG] [app] Starting request: urn:request:8988e352-3e7d-4619-b336-25c7c0d2c6ff (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:40,126 [250] [DEBUG] [app] Ending request: urn:request:8988e352-3e7d-4619-b336-25c7c0d2c6ff (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8988e352-3e7d-4619-b336-25c7c0d2c6ff', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:16:40,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:40,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:40,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:40,129 [261] [DEBUG] [app] Starting request: urn:request:c8504a6c-5e42-40a8-82ef-04588342d40c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:40,129 [261] [DEBUG] [app] Ending request: urn:request:c8504a6c-5e42-40a8-82ef-04588342d40c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c8504a6c-5e42-40a8-82ef-04588342d40c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:40,130 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:40,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:40,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:40,131 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:40,131 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:40,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:40,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:40,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:40,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:40,142 [254] [DEBUG] [app] Ending request: urn:request:d575c906-e837-4f5f-b026-72ba08a6a197 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d575c906-e837-4f5f-b026-72ba08a6a197', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:40,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:40,142 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.021) repositorygcworker stdout | 2025-11-04 09:16:40,181 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:16:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:06.265965+00:00 (in 26.084716 seconds) repositorygcworker stdout | 2025-11-04 09:16:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:40 GMT)" (scheduled at 2025-11-04 09:16:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:16:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:16:40,183 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 40, 182370), True, datetime.datetime(2025, 11, 4, 9, 16, 40, 182370), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:16:40,190 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:16:40,190 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:16:40,190 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:16:40,228 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:16:40,251 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:16:41,032 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:16:41,325 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:16:41,622 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:42,302 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:42,313 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:16:42,316 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:16:42,737 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:43,137 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:16:43,524 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:16:44,339 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:44,805 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:44,819 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:44,830 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:44,837 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:16:44,978 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:16:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:14.980431+00:00 (in 29.999524 seconds) buildlogsarchiver stdout | 2025-11-04 09:16:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:14 GMT)" (scheduled at 2025-11-04 09:16:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:16:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 16, 44, 981155), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:16:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:16:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:16:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:14 GMT)" executed successfully gunicorn-registry stdout | 2025-11-04 09:16:46,304 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,387 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,425 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,454 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,459 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,473 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,477 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,492 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:16:46,493 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:47,681 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:49.676299+00:00 (in 1.995326 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:47,681 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:47 GMT)" (scheduled at 2025-11-04 09:16:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:16:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:57.766253+00:00 (in 9.999558 seconds) notificationworker stdout | 2025-11-04 09:16:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:57 GMT)" (scheduled at 2025-11-04 09:16:47.766253+00:00) notificationworker stdout | 2025-11-04 09:16:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 47, 767090), True, datetime.datetime(2025, 11, 4, 9, 16, 47, 767090), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:47,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:47,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:47,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:16:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:16:59.676299+00:00 (in 9.999407 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:59 GMT)" (scheduled at 2025-11-04 09:16:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 49, 677332), True, datetime.datetime(2025, 11, 4, 9, 16, 49, 677332), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:49,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:49,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:49,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:16:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:16:49,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:16:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:01.707296+00:00 (in 11.999987 seconds) securityworker stdout | 2025-11-04 09:16:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:19 GMT)" (scheduled at 2025-11-04 09:16:49.706478+00:00) securityworker stdout | 2025-11-04 09:16:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:16:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:16:49,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:16:49,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:16:49,718 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:49,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 5, 15]) securityworker stdout | 2025-11-04 09:16:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:16:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:16:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stdout | 2025-11-04 09:16:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stdout | 2025-11-04 09:16:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:16:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:16:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:16:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:16:49,726 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stdout | 2025-11-04 09:16:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stdout | 2025-11-04 09:16:49,726 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-49 securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:16:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 32-42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:16:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 42, 49]) securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 32-42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 32-42 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 32 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-32 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 15-32 securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,730 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-32 securityworker stdout | 2025-11-04 09:16:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-32 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:16:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 19, 29]) securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 15-32 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-32 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Right range 19-29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 19-29 securityworker stdout | 2025-11-04 09:16:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-19 securityworker stdout | 2025-11-04 09:16:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-19 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:16:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 15, 25]) securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Merging with block 19-29 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 29-32 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 19-29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 29-32 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-32 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-32 securityworker stdout | 2025-11-04 09:16:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stdout | 2025-11-04 09:16:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 29, 39]) securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 39-29 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:16:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 39-29 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 18, 28]) securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,739 [92] [DEBUG] [util.migrate.allocator] Left range 18-28 securityworker stdout | 2025-11-04 09:16:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stdout | 2025-11-04 09:16:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 31, 41]) securityworker stdout | 2025-11-04 09:16:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:16:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:16:49,741 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stdout | 2025-11-04 09:16:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-18 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 8 securityworker stdout | 2025-11-04 09:16:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 4, 14]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:16:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 4-14 by worker securityworker stdout | 2025-11-04 09:16:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 4-14 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-14 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-14 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-18 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 14-18 securityworker stdout | 2025-11-04 09:16:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:16:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 14, 24]) securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-14 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-14 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 18-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-28 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Left range 4-28 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-31 securityworker stdout | 2025-11-04 09:16:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 28-31 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 28 securityworker stdout | 2025-11-04 09:16:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 28, 38]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stdout | 2025-11-04 09:16:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 28-38 by worker securityworker stdout | 2025-11-04 09:16:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 28-38 securityworker stdout | 2025-11-04 09:16:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-28 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-28 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-28 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-41 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Left range 4-41 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:16:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 41, 49]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 4-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 4-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-41 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-41 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 4 securityworker stdout | 2025-11-04 09:16:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 1-4 securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 4-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 4-41 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 4 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-4 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-4 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 1, 11]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,754 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 5, 15]) securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:16:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:16:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 1, 11]) securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-49 securityworker stdout | 2025-11-04 09:16:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 27, 37]) securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-27 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-27 securityworker stdout | 2025-11-04 09:16:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:16:49,762 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 16, 26]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-16 securityworker stdout | 2025-11-04 09:16:49,763 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-16 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:16:49,764 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 15, 25]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:16:49,765 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:16:49,765 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:16:49,766 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stdout | 2025-11-04 09:16:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 26 securityworker stdout | 2025-11-04 09:16:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Total range: 26-49 securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-27 securityworker stdout | 2025-11-04 09:16:49,767 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 26-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Right range 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-27 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:16:49,768 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 26, 36]) securityworker stdout | 2025-11-04 09:16:49,768 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:16:49,768 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:16:49,768 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 27-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 37 securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 37 securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stdout | 2025-11-04 09:16:49,769 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:16:49,770 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 38, 48]) securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 38-48 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 38-48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 38-48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 37-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stdout | 2025-11-04 09:16:49,771 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Left range 38-48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 48-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 48 securityworker stdout | 2025-11-04 09:16:49,772 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 48, 49]) securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Total range: 37-38 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 48-49 by worker securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 48-49 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 38-48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 38-48 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 38 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 37-38 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stdout | 2025-11-04 09:16:49,773 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 37-38 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 37 securityworker stdout | 2025-11-04 09:16:49,774 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 11, 49, 711741), 37, 47]) securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:16:49,776 [92] [DEBUG] [util.migrate.allocator] Total range: 47-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-37 securityworker stderr | 2025-11-04 09:16:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:49,777 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:16:49,777 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:16:49,777 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:16:51,154 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:00.086435+00:00 (in 7.926104 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:52 GMT)" (scheduled at 2025-11-04 09:16:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,169 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,169 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:16:52,169 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:16:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:16:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:32.561493+00:00 (in 39.000774 seconds) namespacegcworker stdout | 2025-11-04 09:16:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:53 GMT)" (scheduled at 2025-11-04 09:16:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:16:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:16:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 53, 561089), True, datetime.datetime(2025, 11, 4, 9, 16, 53, 561089), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:16:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:16:53,570 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:16:53,570 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:17:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:16:55,066 [254] [DEBUG] [app] Starting request: urn:request:088305b8-44a9-47e2-9374-a17b819748e4 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:16:55,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:16:55,070 [250] [DEBUG] [app] Starting request: urn:request:4616687a-4040-448d-82f5-1a9af6606d08 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:55,070 [250] [DEBUG] [app] Ending request: urn:request:4616687a-4040-448d-82f5-1a9af6606d08 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:4616687a-4040-448d-82f5-1a9af6606d08', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:55,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:16:55,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:55,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:55,074 [254] [DEBUG] [app] Starting request: urn:request:37e9966e-8454-46ea-9de3-7f76ea33b615 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:55,074 [254] [DEBUG] [app] Ending request: urn:request:37e9966e-8454-46ea-9de3-7f76ea33b615 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:37e9966e-8454-46ea-9de3-7f76ea33b615', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:16:55,075 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:55,076 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:16:55,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:55,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:55,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:55,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:55,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:55,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:55,088 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:55,089 [254] [DEBUG] [app] Ending request: urn:request:088305b8-44a9-47e2-9374-a17b819748e4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:088305b8-44a9-47e2-9374-a17b819748e4', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:55,089 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:55,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.025) gunicorn-web stdout | 2025-11-04 09:16:55,123 [254] [DEBUG] [app] Starting request: urn:request:4df30012-ad48-4e91-a3d9-4d3f70a05c18 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:16:55,132 [256] [DEBUG] [app] Starting request: urn:request:51de70e8-9c0b-46cc-adb2-0c6b50743c15 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:16:55,132 [256] [DEBUG] [app] Ending request: urn:request:51de70e8-9c0b-46cc-adb2-0c6b50743c15 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:51de70e8-9c0b-46cc-adb2-0c6b50743c15', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:16:55,133 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:55,133 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:16:55,134 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:16:55,136 [261] [DEBUG] [app] Starting request: urn:request:c9277f80-3b5b-4bcd-ba02-d799a3175238 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:16:55,137 [261] [DEBUG] [app] Ending request: urn:request:c9277f80-3b5b-4bcd-ba02-d799a3175238 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c9277f80-3b5b-4bcd-ba02-d799a3175238', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:16:55,137 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:16:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:16:55,137 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:16:55,138 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:16:55,138 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:16:55,138 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:16:55,143 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:16:55,143 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:16:55,147 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:16:55,149 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:16:55,150 [254] [DEBUG] [app] Ending request: urn:request:4df30012-ad48-4e91-a3d9-4d3f70a05c18 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4df30012-ad48-4e91-a3d9-4d3f70a05c18', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:16:55,150 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:16:55,151 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:16:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:16:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.029 118 0.029) gcworker stdout | 2025-11-04 09:16:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:16:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:07.663502+00:00 (in 12.001315 seconds) gcworker stdout | 2025-11-04 09:16:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:25 GMT)" (scheduled at 2025-11-04 09:16:55.661677+00:00) gcworker stdout | 2025-11-04 09:16:55,664 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038215662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:16:55,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:16:55,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:16:55,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:16:55,855 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:16:56,243 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:16:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:16:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:07.766253+00:00 (in 9.999543 seconds) notificationworker stdout | 2025-11-04 09:16:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:07 GMT)" (scheduled at 2025-11-04 09:16:57.766253+00:00) notificationworker stdout | 2025-11-04 09:16:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:16:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 57, 767071), True, datetime.datetime(2025, 11, 4, 9, 16, 57, 767071), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:16:57,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:16:57,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:16:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:16:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:16:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:09.676299+00:00 (in 9.999467 seconds) proxycacheblobworker stdout | 2025-11-04 09:16:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:09 GMT)" (scheduled at 2025-11-04 09:16:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:16:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:16:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 16, 59, 677264), True, datetime.datetime(2025, 11, 4, 9, 16, 59, 677264), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:16:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:16:59,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:16:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:09 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:17:00,069 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:52.159714+00:00 (in 52.072746 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:00 GMT)" (scheduled at 2025-11-04 09:17:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,095 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:17:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:00 GMT)" executed successfully globalpromstats stdout | 2025-11-04 09:17:00,167 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:00,451 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:17:01,271 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:17:01,444 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:17:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:19.706478+00:00 (in 17.998621 seconds) securityworker stdout | 2025-11-04 09:17:01,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:31 GMT)" (scheduled at 2025-11-04 09:17:01.707296+00:00) securityworker stdout | 2025-11-04 09:17:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:01,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:01,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:01,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:01,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:01,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 1, 714022), 1, 49]) securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:01,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:01,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:01,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:01,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 1, 714022), 1, 49]) securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:01,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:01,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:01,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:01,730 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:01,730 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:01,730 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:17:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:01,730 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:17:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:17:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:32.081579+00:00 (in 29.999474 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:17:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:32 GMT)" (scheduled at 2025-11-04 09:17:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:17:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:17:02,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:17:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:17:02,653 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:17:04,264 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:17:04,357 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:17:05,453 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:17:05,946 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:17:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:17:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:40.180734+00:00 (in 33.914155 seconds) repositorygcworker stdout | 2025-11-04 09:17:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:06 GMT)" (scheduled at 2025-11-04 09:17:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:17:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:17:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:28.569177+00:00 (in 21.999086 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:06 GMT)" (scheduled at 2025-11-04 09:17:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:17:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:17:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:17:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:37.077358+00:00 (in 29.999485 seconds) autopruneworker stdout | 2025-11-04 09:17:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:37 GMT)" (scheduled at 2025-11-04 09:17:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:17:07,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244227083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:17:07,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:17:07,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:17:07,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:17:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:25.661677+00:00 (in 17.997687 seconds) gcworker stdout | 2025-11-04 09:17:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:37 GMT)" (scheduled at 2025-11-04 09:17:07.663502+00:00) gcworker stdout | 2025-11-04 09:17:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:17:07,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247527671, None, 1, 0]) gcworker stdout | 2025-11-04 09:17:07,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:07,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:17.766253+00:00 (in 9.999503 seconds) notificationworker stdout | 2025-11-04 09:17:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:17 GMT)" (scheduled at 2025-11-04 09:17:07.766253+00:00) notificationworker stdout | 2025-11-04 09:17:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 7, 767185), True, datetime.datetime(2025, 11, 4, 9, 17, 7, 767185), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:17:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:17:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:07.770322+00:00 (in 59.999504 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:17:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:07 GMT)" (scheduled at 2025-11-04 09:17:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:17:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:17:07,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:07,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:07,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:17:07,778 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:17:07,778 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:17:08,461 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:17:09,557 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:19.676299+00:00 (in 9.999395 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:19 GMT)" (scheduled at 2025-11-04 09:17:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 9, 677240), True, datetime.datetime(2025, 11, 4, 9, 17, 9, 677240), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:09,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:09,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:09,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:17:09,821 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:10,066 [254] [DEBUG] [app] Starting request: urn:request:62fa05c8-b769-4df4-ad55-7e9a79d8114b (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:10,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:10,070 [256] [DEBUG] [app] Starting request: urn:request:7758bb7c-4f60-47bc-9b81-c6982152fa66 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:10,070 [256] [DEBUG] [app] Ending request: urn:request:7758bb7c-4f60-47bc-9b81-c6982152fa66 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7758bb7c-4f60-47bc-9b81-c6982152fa66', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:17:10,071 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:10,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:10,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:10,073 [254] [DEBUG] [app] Starting request: urn:request:79fea761-af82-4203-8516-8fa2cc74d47c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:10,074 [254] [DEBUG] [app] Ending request: urn:request:79fea761-af82-4203-8516-8fa2cc74d47c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:79fea761-af82-4203-8516-8fa2cc74d47c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:10,074 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:10,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:10,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:10,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:10,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:10,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:10,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:10,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:10,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:10,088 [254] [DEBUG] [app] Ending request: urn:request:62fa05c8-b769-4df4-ad55-7e9a79d8114b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:62fa05c8-b769-4df4-ad55-7e9a79d8114b', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:10,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:10,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:17:10,122 [254] [DEBUG] [app] Starting request: urn:request:9517ce47-6e52-44b7-adb4-4c662f75fd56 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:17:10,124 [250] [DEBUG] [app] Starting request: urn:request:86c77641-5061-4faf-b489-d1402c5a3646 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:10,125 [250] [DEBUG] [app] Ending request: urn:request:86c77641-5061-4faf-b489-d1402c5a3646 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:86c77641-5061-4faf-b489-d1402c5a3646', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 162 0.002) gunicorn-registry stdout | 2025-11-04 09:17:10,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:10,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:10,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:10,128 [261] [DEBUG] [app] Starting request: urn:request:e0b7026d-e7e8-4c21-8ba8-de68b7fc1d9f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:10,129 [261] [DEBUG] [app] Ending request: urn:request:e0b7026d-e7e8-4c21-8ba8-de68b7fc1d9f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e0b7026d-e7e8-4c21-8ba8-de68b7fc1d9f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:10,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:10,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:10,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:10,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:10,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:10,134 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:10,134 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:10,139 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:10,140 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:10,142 [254] [DEBUG] [app] Ending request: urn:request:9517ce47-6e52-44b7-adb4-4c662f75fd56 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9517ce47-6e52-44b7-adb4-4c662f75fd56', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:10,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:10,142 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) securityworker stdout | 2025-11-04 09:17:10,246 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:17:10,267 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:17:11,049 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:17:11,341 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:17:11,640 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:12,321 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:12,333 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:12,337 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:17:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:17:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:38.480592+00:00 (in 25.997958 seconds) exportactionlogsworker stdout | 2025-11-04 09:17:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:12 GMT)" (scheduled at 2025-11-04 09:17:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:17:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:17:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:17:12,754 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:13,155 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:17:13,543 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:14,356 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:14,826 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:14,841 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:14,849 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:14,876 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:17:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:44.980431+00:00 (in 29.999491 seconds) buildlogsarchiver stdout | 2025-11-04 09:17:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:44 GMT)" (scheduled at 2025-11-04 09:17:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:17:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 17, 14, 981263), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:17:14,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:17:14,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:17:14,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:14,997 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,327 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,408 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,444 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,478 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,492 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,494 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,515 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,522 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:16,503 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:19.767846+00:00 (in 2.001064 seconds) notificationworker stdout | 2025-11-04 09:17:17,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:27 GMT)" (scheduled at 2025-11-04 09:17:17.766253+00:00) notificationworker stdout | 2025-11-04 09:17:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 17, 767213), True, datetime.datetime(2025, 11, 4, 9, 17, 17, 767213), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:17,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:17,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:17:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:29.676299+00:00 (in 9.999565 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:29 GMT)" (scheduled at 2025-11-04 09:17:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 19, 677179), True, datetime.datetime(2025, 11, 4, 9, 17, 19, 677179), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:17:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:31.707296+00:00 (in 12.000319 seconds) securityworker stdout | 2025-11-04 09:17:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:49 GMT)" (scheduled at 2025-11-04 09:17:19.706478+00:00) securityworker stdout | 2025-11-04 09:17:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:19,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:19,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:17:19,717 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:19,717 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:49 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:19,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:19,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:27.766253+00:00 (in 7.997980 seconds) notificationworker stdout | 2025-11-04 09:17:19,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:19 GMT)" (scheduled at 2025-11-04 09:17:19.767846+00:00) notificationworker stdout | 2025-11-04 09:17:19,769 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), 'notification/%']) notificationworker stdout | 2025-11-04 09:17:19,776 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), True, datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:17:19,778 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), True, datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 17, 19, 768709), 'notification/%']) notificationworker stdout | 2025-11-04 09:17:19,779 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:19,779 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:17:21,171 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:25,066 [258] [DEBUG] [app] Starting request: urn:request:92a13da7-c802-4cef-a77c-58e25d0d93e5 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:25,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:25,070 [260] [DEBUG] [app] Starting request: urn:request:d410bba8-791e-4843-b352-853569af9496 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:25,070 [260] [DEBUG] [app] Ending request: urn:request:d410bba8-791e-4843-b352-853569af9496 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:d410bba8-791e-4843-b352-853569af9496', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:25,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:25,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:25,072 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:25,074 [254] [DEBUG] [app] Starting request: urn:request:49fc6baf-48c7-4bf1-a674-91fd057439ad (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:25,075 [254] [DEBUG] [app] Ending request: urn:request:49fc6baf-48c7-4bf1-a674-91fd057439ad (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:49fc6baf-48c7-4bf1-a674-91fd057439ad', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:25,075 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:25,075 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:25,076 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:25,076 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:25,076 [258] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.003) gunicorn-web stdout | 2025-11-04 09:17:25,081 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:25,081 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:25,086 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:25,088 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:25,089 [258] [DEBUG] [app] Ending request: urn:request:92a13da7-c802-4cef-a77c-58e25d0d93e5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:92a13da7-c802-4cef-a77c-58e25d0d93e5', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:25,089 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:25,089 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.025) gunicorn-web stdout | 2025-11-04 09:17:25,122 [254] [DEBUG] [app] Starting request: urn:request:8f369fe2-99b3-4d2a-ba7b-e71bfe7629a1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:25,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:25,129 [260] [DEBUG] [app] Starting request: urn:request:97e5cb72-aba9-462a-997b-1591484ee8f7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:25,129 [260] [DEBUG] [app] Ending request: urn:request:97e5cb72-aba9-462a-997b-1591484ee8f7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:97e5cb72-aba9-462a-997b-1591484ee8f7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:17:25,130 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:25,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:25,131 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:25,132 [261] [DEBUG] [app] Starting request: urn:request:a0113827-3127-4bb0-816c-7394cb9624e6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:25,133 [261] [DEBUG] [app] Ending request: urn:request:a0113827-3127-4bb0-816c-7394cb9624e6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a0113827-3127-4bb0-816c-7394cb9624e6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:25,133 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:25,133 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:25,134 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:25,134 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:25,134 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:25,139 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:25,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:25,145 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:25,146 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:25,147 [254] [DEBUG] [app] Ending request: urn:request:8f369fe2-99b3-4d2a-ba7b-e71bfe7629a1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8f369fe2-99b3-4d2a-ba7b-e71bfe7629a1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:25,147 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:25,148 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.026) gcworker stdout | 2025-11-04 09:17:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:37.663502+00:00 (in 12.001298 seconds) gcworker stdout | 2025-11-04 09:17:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:55 GMT)" (scheduled at 2025-11-04 09:17:25.661677+00:00) gcworker stdout | 2025-11-04 09:17:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038245662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:17:25,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:17:25,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:25,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:17:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:25,872 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:26,260 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:27.768179+00:00 (in 0.001487 seconds) notificationworker stdout | 2025-11-04 09:17:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:37 GMT)" (scheduled at 2025-11-04 09:17:27.766253+00:00) notificationworker stdout | 2025-11-04 09:17:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 27, 767095), True, datetime.datetime(2025, 11, 4, 9, 17, 27, 767095), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:37.766253+00:00 (in 9.997403 seconds) notificationworker stdout | 2025-11-04 09:17:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:27 GMT)" (scheduled at 2025-11-04 09:17:27.768179+00:00) notificationworker stdout | 2025-11-04 09:17:27,769 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:17:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:37 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:28,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:28,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:33.568037+00:00 (in 4.998311 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:28,569 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:28 GMT)" (scheduled at 2025-11-04 09:17:28.569177+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:28,570 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:28,579 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), True, datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:28,581 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), True, datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 17, 28, 570016), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:17:28,583 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:17:28,583 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:28 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:39.676299+00:00 (in 9.999461 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:29,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:39 GMT)" (scheduled at 2025-11-04 09:17:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 29, 677282), True, datetime.datetime(2025, 11, 4, 9, 17, 29, 677282), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:29,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:29,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:29,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:17:30,086 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:17:30,184 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:17:30,468 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:17:31,287 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:17:31,477 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:17:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:49.706478+00:00 (in 17.998629 seconds) securityworker stdout | 2025-11-04 09:17:31,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:01 GMT)" (scheduled at 2025-11-04 09:17:31.707296+00:00) securityworker stdout | 2025-11-04 09:17:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:31,712 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:31,715 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:31,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:31,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:31,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 31, 715601), 1, 49]) securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:31,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:31,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:31,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:31,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:31,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 31, 715601), 1, 49]) securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:31,733 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:17:31,733 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:01 GMT)" executed successfully securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:17:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker pullstatsredisflushworker stdout | 2025-11-04 09:17:32,082 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:17:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:02.081579+00:00 (in 29.999264 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:17:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:02 GMT)" (scheduled at 2025-11-04 09:17:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:17:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:17:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:17:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:02 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:17:32,561 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:17:32,561 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:33.561829+00:00 (in 0.999857 seconds) namespacegcworker stdout | 2025-11-04 09:17:32,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:32 GMT)" (scheduled at 2025-11-04 09:17:32.561493+00:00) namespacegcworker stdout | 2025-11-04 09:17:32,563 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:17:32,570 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), True, datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), 0, 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:17:32,571 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), True, datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 17, 32, 562380), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:17:32,573 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:17:32,574 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:22:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:17:32,669 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:17:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:53.560131+00:00 (in 19.997709 seconds) namespacegcworker stdout | 2025-11-04 09:17:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" (scheduled at 2025-11-04 09:17:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:17:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:17:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:17:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:17:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:06.569529+00:00 (in 33.000970 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:17:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" (scheduled at 2025-11-04 09:17:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:17:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:17:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 33, 568982), True, datetime.datetime(2025, 11, 4, 9, 17, 33, 568982), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:17:33,579 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:17:33,579 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:17:33,579 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:17:34,280 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:17:34,374 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:17:35,469 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:17:35,963 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:17:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:17:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:07.077358+00:00 (in 29.999501 seconds) autopruneworker stdout | 2025-11-04 09:17:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:07 GMT)" (scheduled at 2025-11-04 09:17:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:17:37,085 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244257084, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:17:37,089 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:17:37,089 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:17:37,089 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:17:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:55.661677+00:00 (in 17.997549 seconds) gcworker stdout | 2025-11-04 09:17:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:07 GMT)" (scheduled at 2025-11-04 09:17:37.663502+00:00) gcworker stdout | 2025-11-04 09:17:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:17:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247557671, None, 1, 0]) gcworker stdout | 2025-11-04 09:17:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:47.766253+00:00 (in 9.999503 seconds) notificationworker stdout | 2025-11-04 09:17:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:47 GMT)" (scheduled at 2025-11-04 09:17:37.766253+00:00) notificationworker stdout | 2025-11-04 09:17:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 37, 767081), True, datetime.datetime(2025, 11, 4, 9, 17, 37, 767081), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:37,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:37,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:37,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:47 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:17:38,478 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:17:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:17:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:12.482065+00:00 (in 34.000980 seconds) exportactionlogsworker stdout | 2025-11-04 09:17:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:38 GMT)" (scheduled at 2025-11-04 09:17:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:17:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:17:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 38, 481569), True, datetime.datetime(2025, 11, 4, 9, 17, 38, 481569), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:17:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:17:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:17:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:38 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:17:39,574 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:47.679968+00:00 (in 8.003101 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:39,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:49 GMT)" (scheduled at 2025-11-04 09:17:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 39, 677247), True, datetime.datetime(2025, 11, 4, 9, 17, 39, 677247), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:39,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:39,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:39,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:17:39,854 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:40,065 [261] [DEBUG] [app] Starting request: urn:request:2bd8d8ac-f665-42c2-b5a4-b2ecfaf1de6e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:40,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:40,068 [250] [DEBUG] [app] Starting request: urn:request:0c1abb93-3f5e-46f1-b84d-365d95dbb4de (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:40,070 [250] [DEBUG] [app] Ending request: urn:request:0c1abb93-3f5e-46f1-b84d-365d95dbb4de (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0c1abb93-3f5e-46f1-b84d-365d95dbb4de', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:40,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:40,070 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:40,071 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:40,073 [254] [DEBUG] [app] Starting request: urn:request:e969af82-1281-43a7-a5cd-d922decd914d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:40,074 [254] [DEBUG] [app] Ending request: urn:request:e969af82-1281-43a7-a5cd-d922decd914d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e969af82-1281-43a7-a5cd-d922decd914d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:40,074 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:40,074 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:40,075 [261] [DEBUG] [data.model.health] Validating database connection. nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:40,075 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:40,075 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:40,080 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:40,081 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:40,086 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:40,087 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:40,088 [261] [DEBUG] [app] Ending request: urn:request:2bd8d8ac-f665-42c2-b5a4-b2ecfaf1de6e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2bd8d8ac-f665-42c2-b5a4-b2ecfaf1de6e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:40,088 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:40,089 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:17:40,123 [254] [DEBUG] [app] Starting request: urn:request:260662fd-3155-4ecd-96fd-c9dc23fc26c4 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:40,125 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:40,126 [256] [DEBUG] [app] Starting request: urn:request:7f17ba36-f2ef-4171-a2ca-be05d4d92491 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:40,127 [256] [DEBUG] [app] Ending request: urn:request:7f17ba36-f2ef-4171-a2ca-be05d4d92491 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7f17ba36-f2ef-4171-a2ca-be05d4d92491', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:40,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:40,129 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:40,129 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:40,131 [261] [DEBUG] [app] Starting request: urn:request:3a5f77cf-c173-4f20-9ab3-86308a4a249e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:40,131 [261] [DEBUG] [app] Ending request: urn:request:3a5f77cf-c173-4f20-9ab3-86308a4a249e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3a5f77cf-c173-4f20-9ab3-86308a4a249e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:40,131 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:40,132 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:40,132 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:40,132 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:40,133 [254] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:17:40,138 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:40,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:40,143 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:40,144 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:40,145 [254] [DEBUG] [app] Ending request: urn:request:260662fd-3155-4ecd-96fd-c9dc23fc26c4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:260662fd-3155-4ecd-96fd-c9dc23fc26c4', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:40,145 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:40,145 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.023) repositorygcworker stdout | 2025-11-04 09:17:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:17:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:06.265965+00:00 (in 26.084761 seconds) repositorygcworker stdout | 2025-11-04 09:17:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:40 GMT)" (scheduled at 2025-11-04 09:17:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:17:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:17:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 40, 181951), True, datetime.datetime(2025, 11, 4, 9, 17, 40, 181951), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:17:40,189 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:17:40,190 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:17:40,190 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:17:40,262 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:17:40,284 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:17:41,067 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:17:41,357 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:17:41,674 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:42,339 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:42,353 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:17:42,355 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:17:42,771 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:43,173 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:17:43,574 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:17:44,372 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:44,849 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:44,869 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:44,872 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:17:44,896 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:17:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:14.980431+00:00 (in 29.999510 seconds) buildlogsarchiver stdout | 2025-11-04 09:17:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:14 GMT)" (scheduled at 2025-11-04 09:17:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:17:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 17, 44, 981264), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:17:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:17:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:17:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:45,018 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,363 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,430 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,465 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,500 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,515 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,515 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,535 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,544 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:17:46,545 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:49.676299+00:00 (in 1.995611 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:47 GMT)" (scheduled at 2025-11-04 09:17:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:17:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:57.766253+00:00 (in 9.999453 seconds) notificationworker stdout | 2025-11-04 09:17:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:57 GMT)" (scheduled at 2025-11-04 09:17:47.766253+00:00) notificationworker stdout | 2025-11-04 09:17:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 47, 767121), True, datetime.datetime(2025, 11, 4, 9, 17, 47, 767121), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:47,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:47,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:47,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:17:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:17:59.676299+00:00 (in 9.999419 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:59 GMT)" (scheduled at 2025-11-04 09:17:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 49, 677275), True, datetime.datetime(2025, 11, 4, 9, 17, 49, 677275), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:49,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:49,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:49,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:17:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:17:49,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:17:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:01.707296+00:00 (in 12.000004 seconds) securityworker stdout | 2025-11-04 09:17:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:19 GMT)" (scheduled at 2025-11-04 09:17:49.706478+00:00) securityworker stdout | 2025-11-04 09:17:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:17:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:17:49,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:17:49,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:17:49,719 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 16, 26]) securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-49 securityworker stdout | 2025-11-04 09:17:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 37, 47]) securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-37 securityworker stdout | 2025-11-04 09:17:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-37 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:17:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 26, 36]) securityworker stdout | 2025-11-04 09:17:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:17:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:17:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stdout | 2025-11-04 09:17:49,730 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stdout | 2025-11-04 09:17:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-26 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Left range 16-36 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-37 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-37 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 36 securityworker stdout | 2025-11-04 09:17:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 36, 46]) securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-36 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-36 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-47 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,733 [92] [DEBUG] [util.migrate.allocator] Left range 16-47 securityworker stdout | 2025-11-04 09:17:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:17:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 16-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:17:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 47, 49]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-47 securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-47 securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 16-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 16-47 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 16 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-16 securityworker stdout | 2025-11-04 09:17:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 11-16 securityworker stdout | 2025-11-04 09:17:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-16 securityworker stdout | 2025-11-04 09:17:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-16 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:17:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 11, 21]) securityworker stdout | 2025-11-04 09:17:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 10, 20]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stdout | 2025-11-04 09:17:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-49 securityworker stdout | 2025-11-04 09:17:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 29, 39]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:17:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:17:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:17:49,741 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 29-39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stdout | 2025-11-04 09:17:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 1, 11]) securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 20-49 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 29-39 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-29 securityworker stdout | 2025-11-04 09:17:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 20-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 29-39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-29 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:17:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 20, 30]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Merging with block 29-39 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Total range: 39-49 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 29-39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 39-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 39-49 securityworker stdout | 2025-11-04 09:17:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 39, 49]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:17:49,748 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 39-49 by worker securityworker stdout | 2025-11-04 09:17:49,748 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stdout | 2025-11-04 09:17:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stdout | 2025-11-04 09:17:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 49-39 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 39-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:17:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 10, 20]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 10-20 by worker securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stdout | 2025-11-04 09:17:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 26, 36]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 26-36 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Right range 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-26 securityworker stdout | 2025-11-04 09:17:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 20-26 securityworker stdout | 2025-11-04 09:17:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 20 securityworker stdout | 2025-11-04 09:17:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 20, 30]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-20 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-20 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 26-36 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-36 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-20 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 26-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Left range 10-36 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-49 securityworker stdout | 2025-11-04 09:17:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Left range 10-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 36-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:17:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 36, 46]) securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-36 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-36 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 36-46 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 36-46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 10-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 10-36 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-46 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 10-46 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:17:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:17:49,760 [92] [DEBUG] [util.migrate.allocator] Right range 10-46 securityworker stdout | 2025-11-04 09:17:49,760 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Right range 10-46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-10 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:49,760 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:17:49,761 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 1, 11]) securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-46 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 46 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Total range: 46-49 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stdout | 2025-11-04 09:17:49,762 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 10-46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 46-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 46-49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 46 securityworker stdout | 2025-11-04 09:17:49,763 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 12, 49, 713777), 46, 49]) securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 46-49 by worker securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 46-49 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 46 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] Total range: 49-46 securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:49,764 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-46 securityworker stderr | 2025-11-04 09:17:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:17:49,764 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:17:51,191 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:00.086435+00:00 (in 7.925989 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:52 GMT)" (scheduled at 2025-11-04 09:17:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:17:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:17:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:17:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:33.561829+00:00 (in 40.001106 seconds) namespacegcworker stdout | 2025-11-04 09:17:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:53 GMT)" (scheduled at 2025-11-04 09:17:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:17:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:17:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 53, 561157), True, datetime.datetime(2025, 11, 4, 9, 17, 53, 561157), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:17:53,569 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:17:53,569 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:17:53,569 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:18:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:17:55,065 [261] [DEBUG] [app] Starting request: urn:request:d0e2237b-8a13-41fd-b8f2-cb82e7beb350 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:17:55,075 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:17:55,077 [250] [DEBUG] [app] Starting request: urn:request:68502357-9b8c-471f-9e92-0887f056c121 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:55,078 [250] [DEBUG] [app] Ending request: urn:request:68502357-9b8c-471f-9e92-0887f056c121 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:68502357-9b8c-471f-9e92-0887f056c121', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:55,078 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:55,079 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:17:55,080 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:55,082 [254] [DEBUG] [app] Starting request: urn:request:998b9f9f-95a2-4607-a5f4-e3aa2bd5853f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:55,083 [254] [DEBUG] [app] Ending request: urn:request:998b9f9f-95a2-4607-a5f4-e3aa2bd5853f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:998b9f9f-95a2-4607-a5f4-e3aa2bd5853f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:55,083 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:55,084 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:55,084 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:55,084 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:55,084 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:55,090 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:55,090 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:55,095 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:55,097 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:55,097 [261] [DEBUG] [app] Ending request: urn:request:d0e2237b-8a13-41fd-b8f2-cb82e7beb350 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d0e2237b-8a13-41fd-b8f2-cb82e7beb350', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:55,098 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:55,098 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.034 118 0.034) gunicorn-web stdout | 2025-11-04 09:17:55,122 [261] [DEBUG] [app] Starting request: urn:request:5346120d-78c0-48e5-818e-76782d1b8436 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:17:55,124 [250] [DEBUG] [app] Starting request: urn:request:e30bd4b6-865e-476c-a21f-19594486bd3a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:17:55,125 [250] [DEBUG] [app] Ending request: urn:request:e30bd4b6-865e-476c-a21f-19594486bd3a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e30bd4b6-865e-476c-a21f-19594486bd3a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:17:55,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:55,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:17:55,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:17:55,128 [254] [DEBUG] [app] Starting request: urn:request:44a990d9-414f-45ee-9f24-a75ae985ef68 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:17:55,129 [254] [DEBUG] [app] Ending request: urn:request:44a990d9-414f-45ee-9f24-a75ae985ef68 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:44a990d9-414f-45ee-9f24-a75ae985ef68', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:17:55,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:17:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:17:55,130 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:17:55,130 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:17:55,130 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:17:55,130 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:17:55,136 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:17:55,136 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:17:55,141 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:17:55,142 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:17:55,143 [261] [DEBUG] [app] Ending request: urn:request:5346120d-78c0-48e5-818e-76782d1b8436 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5346120d-78c0-48e5-818e-76782d1b8436', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:17:55,143 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:17:55,144 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:17:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:17:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) gcworker stdout | 2025-11-04 09:17:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:17:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:07.663502+00:00 (in 12.001278 seconds) gcworker stdout | 2025-11-04 09:17:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:25 GMT)" (scheduled at 2025-11-04 09:17:55.661677+00:00) gcworker stdout | 2025-11-04 09:17:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038275662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:17:55,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:17:55,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:17:55,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:17:55,889 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:17:56,278 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:17:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:17:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:07.766253+00:00 (in 9.999503 seconds) notificationworker stdout | 2025-11-04 09:17:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:07 GMT)" (scheduled at 2025-11-04 09:17:57.766253+00:00) notificationworker stdout | 2025-11-04 09:17:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:17:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 57, 767078), True, datetime.datetime(2025, 11, 4, 9, 17, 57, 767078), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:17:57,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:17:57,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:17:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:17:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:17:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:09.676299+00:00 (in 9.999483 seconds) proxycacheblobworker stdout | 2025-11-04 09:17:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:09 GMT)" (scheduled at 2025-11-04 09:17:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:17:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:17:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 17, 59, 677161), True, datetime.datetime(2025, 11, 4, 9, 17, 59, 677161), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:17:59,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:17:59,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:17:59,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,087 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:52.159714+00:00 (in 52.072536 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:00 GMT)" (scheduled at 2025-11-04 09:18:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,088 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,095 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,096 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:18:00,096 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:18:00,103 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:18:00,199 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:18:00,486 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:18:01,308 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:18:01,496 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:19.706478+00:00 (in 17.998665 seconds) securityworker stdout | 2025-11-04 09:18:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:31 GMT)" (scheduled at 2025-11-04 09:18:01.707296+00:00) securityworker stdout | 2025-11-04 09:18:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:01,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:01,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:01,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:01,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:01,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:01,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:01,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:01,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 1, 714692), 1, 49]) securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:01,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:01,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 1, 714692), 1, 49]) securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:01,732 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:01,733 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:18:02,082 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:18:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:32.081579+00:00 (in 29.999250 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:18:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:32 GMT)" (scheduled at 2025-11-04 09:18:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:18:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:18:02,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:18:02,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:18:02,685 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:18:04,297 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:18:04,391 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:18:05,487 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:18:05,980 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:18:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:18:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:40.180734+00:00 (in 33.914241 seconds) repositorygcworker stdout | 2025-11-04 09:18:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:06 GMT)" (scheduled at 2025-11-04 09:18:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:18:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:18:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:18:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:18:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:33.568037+00:00 (in 26.998045 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:18:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:06 GMT)" (scheduled at 2025-11-04 09:18:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:18:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:18:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:18:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:18:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:37.077358+00:00 (in 29.999487 seconds) autopruneworker stdout | 2025-11-04 09:18:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:37 GMT)" (scheduled at 2025-11-04 09:18:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:18:07,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244287083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:18:07,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:18:07,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:18:07,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:18:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:25.661677+00:00 (in 17.997686 seconds) gcworker stdout | 2025-11-04 09:18:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:37 GMT)" (scheduled at 2025-11-04 09:18:07.663502+00:00) gcworker stdout | 2025-11-04 09:18:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:18:07,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247587672, None, 1, 0]) gcworker stdout | 2025-11-04 09:18:07,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:07,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:07,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:17.766253+00:00 (in 9.999112 seconds) notificationworker stdout | 2025-11-04 09:18:07,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:17 GMT)" (scheduled at 2025-11-04 09:18:07.766253+00:00) notificationworker stdout | 2025-11-04 09:18:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:07,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 7, 767589), True, datetime.datetime(2025, 11, 4, 9, 18, 7, 767589), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:18:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:18:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:07.770322+00:00 (in 59.999495 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:18:07,771 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:07 GMT)" (scheduled at 2025-11-04 09:18:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:18:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:18:07,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:07,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:07,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:18:07,777 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:18:07,777 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:18:08,494 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:18:09,591 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:19.676299+00:00 (in 9.999543 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:19 GMT)" (scheduled at 2025-11-04 09:18:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 9, 677182), True, datetime.datetime(2025, 11, 4, 9, 18, 9, 677182), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:09,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:09,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:09,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:18:09,872 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:10,065 [254] [DEBUG] [app] Starting request: urn:request:ebf8dd1e-2d80-464e-9e77-18d4ff65f88f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:10,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:10,069 [260] [DEBUG] [app] Starting request: urn:request:0964e288-78c8-4c3b-8329-6f33c79d94c3 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:10,069 [260] [DEBUG] [app] Ending request: urn:request:0964e288-78c8-4c3b-8329-6f33c79d94c3 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:0964e288-78c8-4c3b-8329-6f33c79d94c3', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:18:10,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:10,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:10,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:10,073 [258] [DEBUG] [app] Starting request: urn:request:914812c5-259c-47d4-8598-929bbf664fb8 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:10,073 [258] [DEBUG] [app] Ending request: urn:request:914812c5-259c-47d4-8598-929bbf664fb8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:914812c5-259c-47d4-8598-929bbf664fb8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:10,074 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:10,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:10,075 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:10,075 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:10,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:10,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:10,080 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:10,087 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:10,089 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:10,090 [254] [DEBUG] [app] Ending request: urn:request:ebf8dd1e-2d80-464e-9e77-18d4ff65f88f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ebf8dd1e-2d80-464e-9e77-18d4ff65f88f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:10,090 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:10,091 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.027 118 0.026) gunicorn-web stdout | 2025-11-04 09:18:10,123 [254] [DEBUG] [app] Starting request: urn:request:0b4744f7-28f3-4074-8acd-78f57ba9bbf5 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:18:10,125 [260] [DEBUG] [app] Starting request: urn:request:56dfd7a9-ef40-4770-950e-2f3773426695 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:10,125 [260] [DEBUG] [app] Ending request: urn:request:56dfd7a9-ef40-4770-950e-2f3773426695 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:56dfd7a9-ef40-4770-950e-2f3773426695', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:10,126 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:18:10,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:10,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:10,129 [254] [DEBUG] [app] Starting request: urn:request:1f4d149e-58a5-48d9-9b22-cb58cf1c4f17 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:10,129 [254] [DEBUG] [app] Ending request: urn:request:1f4d149e-58a5-48d9-9b22-cb58cf1c4f17 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1f4d149e-58a5-48d9-9b22-cb58cf1c4f17', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:10,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:10,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:10,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:10,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:10,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:10,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:10,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:10,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:10,142 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:10,143 [254] [DEBUG] [app] Ending request: urn:request:0b4744f7-28f3-4074-8acd-78f57ba9bbf5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0b4744f7-28f3-4074-8acd-78f57ba9bbf5', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:10,143 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:10,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) securityworker stdout | 2025-11-04 09:18:10,279 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:18:10,300 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:18:11,084 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:18:11,373 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:18:11,690 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:12,358 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:12,373 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:12,373 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:18:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:18:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:38.480592+00:00 (in 25.998018 seconds) exportactionlogsworker stdout | 2025-11-04 09:18:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:12 GMT)" (scheduled at 2025-11-04 09:18:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:18:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:18:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:18:12,788 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:13,189 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:18:13,591 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:14,390 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:14,869 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:14,887 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:14,896 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:14,914 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:18:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:18:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:44.980431+00:00 (in 29.999527 seconds) buildlogsarchiver stdout | 2025-11-04 09:18:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:44 GMT)" (scheduled at 2025-11-04 09:18:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:18:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 18, 14, 981178), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:18:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:18:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:18:15,039 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,385 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,452 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,486 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,538 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,524 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,539 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,560 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,567 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:16,568 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:27.766253+00:00 (in 9.999510 seconds) notificationworker stdout | 2025-11-04 09:18:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:27 GMT)" (scheduled at 2025-11-04 09:18:17.766253+00:00) notificationworker stdout | 2025-11-04 09:18:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 17, 767155), True, datetime.datetime(2025, 11, 4, 9, 18, 17, 767155), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:17,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:17,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:17,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:18:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:29.676299+00:00 (in 9.999508 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:29 GMT)" (scheduled at 2025-11-04 09:18:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 19, 677199), True, datetime.datetime(2025, 11, 4, 9, 18, 19, 677199), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:19,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:19,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:19,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:18:19,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:19,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:31.707296+00:00 (in 11.999927 seconds) securityworker stdout | 2025-11-04 09:18:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:49 GMT)" (scheduled at 2025-11-04 09:18:19.706478+00:00) securityworker stdout | 2025-11-04 09:18:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:19,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:19,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:18:19,719 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:18:19,719 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:18:21,208 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:25,066 [258] [DEBUG] [app] Starting request: urn:request:b6aa38b2-2a8c-47f5-ba31-504119af3ac9 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:25,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:25,070 [250] [DEBUG] [app] Starting request: urn:request:b4ec6abb-0ba5-4994-99d0-3e3616786c8d (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:25,070 [250] [DEBUG] [app] Ending request: urn:request:b4ec6abb-0ba5-4994-99d0-3e3616786c8d (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b4ec6abb-0ba5-4994-99d0-3e3616786c8d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:25,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:25,071 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 162 0.002) gunicorn-web stdout | 2025-11-04 09:18:25,072 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:25,122 [254] [DEBUG] [app] Starting request: urn:request:377dfc9b-8411-43b0-a8fd-dd799f37f08a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:25,124 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:25,126 [250] [DEBUG] [app] Starting request: urn:request:c357102e-67c1-4d91-b6e7-61b5b097c501 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:25,126 [250] [DEBUG] [app] Ending request: urn:request:c357102e-67c1-4d91-b6e7-61b5b097c501 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c357102e-67c1-4d91-b6e7-61b5b097c501', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:25,127 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:25,127 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:25,128 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:25,130 [257] [DEBUG] [app] Starting request: urn:request:daf085e8-2e7e-496a-bf50-7d948515f4c2 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:25,131 [257] [DEBUG] [app] Ending request: urn:request:daf085e8-2e7e-496a-bf50-7d948515f4c2 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:daf085e8-2e7e-496a-bf50-7d948515f4c2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:25,131 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 159 0.002) gunicorn-web stdout | 2025-11-04 09:18:25,132 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:25,132 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:25,132 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:25,132 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:25,138 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:25,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:25,143 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:25,144 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:25,145 [254] [DEBUG] [app] Ending request: urn:request:377dfc9b-8411-43b0-a8fd-dd799f37f08a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:377dfc9b-8411-43b0-a8fd-dd799f37f08a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:25,145 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:18:25,146 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:18:25,239 [258] [DEBUG] [app] Starting request: urn:request:01b8346d-dca9-4f2d-a4d9-fc3dc92ffdcb (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:25,239 [258] [DEBUG] [app] Ending request: urn:request:01b8346d-dca9-4f2d-a4d9-fc3dc92ffdcb (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:01b8346d-dca9-4f2d-a4d9-fc3dc92ffdcb', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:25,240 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:25,240 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:25,240 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:25,240 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:25,241 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:25,246 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:25,246 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:25,250 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:25,252 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:25,253 [258] [DEBUG] [app] Ending request: urn:request:b6aa38b2-2a8c-47f5-ba31-504119af3ac9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:b6aa38b2-2a8c-47f5-ba31-504119af3ac9', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:25,254 [258] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.189 118 0.189) gunicorn-web stdout | 2025-11-04 09:18:25,255 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gcworker stdout | 2025-11-04 09:18:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:37.663502+00:00 (in 12.001355 seconds) gcworker stdout | 2025-11-04 09:18:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:55 GMT)" (scheduled at 2025-11-04 09:18:25.661677+00:00) gcworker stdout | 2025-11-04 09:18:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038305662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:18:25,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:18:25,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:25,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:18:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:25,905 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:18:26,297 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:27.768179+00:00 (in 0.001454 seconds) notificationworker stdout | 2025-11-04 09:18:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:37 GMT)" (scheduled at 2025-11-04 09:18:27.766253+00:00) notificationworker stdout | 2025-11-04 09:18:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 27, 767145), True, datetime.datetime(2025, 11, 4, 9, 18, 27, 767145), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:37.766253+00:00 (in 9.997612 seconds) notificationworker stdout | 2025-11-04 09:18:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:27 GMT)" (scheduled at 2025-11-04 09:18:27.768179+00:00) notificationworker stdout | 2025-11-04 09:18:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:18:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:39.676299+00:00 (in 9.999444 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:29,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:39 GMT)" (scheduled at 2025-11-04 09:18:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 29, 677276), True, datetime.datetime(2025, 11, 4, 9, 18, 29, 677276), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:29,687 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:29,687 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:29,687 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:18:30,118 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:18:30,217 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:18:30,503 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:18:31,326 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:18:31,513 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:18:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:31,708 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:49.706478+00:00 (in 17.998450 seconds) securityworker stdout | 2025-11-04 09:18:31,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:01 GMT)" (scheduled at 2025-11-04 09:18:31.707296+00:00) securityworker stdout | 2025-11-04 09:18:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:31,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:31,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:31,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:31,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 31, 714660), 1, 49]) securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:31,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:31,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 31, 714660), 1, 49]) securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:31,732 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:18:31,732 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:01 GMT)" executed successfully securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:18:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker pullstatsredisflushworker stdout | 2025-11-04 09:18:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:18:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:02.081579+00:00 (in 29.999453 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:18:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:02 GMT)" (scheduled at 2025-11-04 09:18:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:18:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:18:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:18:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:18:32,701 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:18:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:53.560131+00:00 (in 19.997745 seconds) namespacegcworker stdout | 2025-11-04 09:18:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" (scheduled at 2025-11-04 09:18:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:18:33,563 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:18:33,563 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:18:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:18:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:06.569529+00:00 (in 33.000934 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:18:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" (scheduled at 2025-11-04 09:18:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:18:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:18:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 33, 568885), True, datetime.datetime(2025, 11, 4, 9, 18, 33, 568885), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:18:33,578 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:18:33,578 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:18:33,578 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:18:34,313 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:18:34,407 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:18:35,504 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:18:35,995 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:18:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:18:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:07.077358+00:00 (in 29.999432 seconds) autopruneworker stdout | 2025-11-04 09:18:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:07 GMT)" (scheduled at 2025-11-04 09:18:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:18:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244317083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:18:37,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:18:37,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:18:37,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:18:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:55.661677+00:00 (in 17.997650 seconds) gcworker stdout | 2025-11-04 09:18:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:07 GMT)" (scheduled at 2025-11-04 09:18:37.663502+00:00) gcworker stdout | 2025-11-04 09:18:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:18:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247617670, None, 1, 0]) gcworker stdout | 2025-11-04 09:18:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:37,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:47.766253+00:00 (in 9.999122 seconds) notificationworker stdout | 2025-11-04 09:18:37,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:47 GMT)" (scheduled at 2025-11-04 09:18:37.766253+00:00) notificationworker stdout | 2025-11-04 09:18:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 37, 767596), True, datetime.datetime(2025, 11, 4, 9, 18, 37, 767596), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:37,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:37,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:37,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:18:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:18:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:12.482065+00:00 (in 34.000976 seconds) exportactionlogsworker stdout | 2025-11-04 09:18:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:38 GMT)" (scheduled at 2025-11-04 09:18:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:18:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:18:38,483 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 38, 481554), True, datetime.datetime(2025, 11, 4, 9, 18, 38, 481554), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:18:38,491 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:18:38,491 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:18:38,491 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:18:38,510 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:18:39,609 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:47.679968+00:00 (in 8.003139 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:39,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:49 GMT)" (scheduled at 2025-11-04 09:18:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 39, 677230), True, datetime.datetime(2025, 11, 4, 9, 18, 39, 677230), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:39,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:39,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:39,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:18:39,891 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:40,066 [254] [DEBUG] [app] Starting request: urn:request:c06e996b-86c0-41b0-a3fa-c479580dd0a6 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:40,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:40,070 [250] [DEBUG] [app] Starting request: urn:request:e230c269-5483-464c-a282-c12830c32759 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:40,070 [250] [DEBUG] [app] Ending request: urn:request:e230c269-5483-464c-a282-c12830c32759 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e230c269-5483-464c-a282-c12830c32759', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:18:40,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:40,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:40,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:40,077 [257] [DEBUG] [app] Starting request: urn:request:009a4062-c638-4c9f-847d-d6aaacb15be8 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:40,078 [257] [DEBUG] [app] Ending request: urn:request:009a4062-c638-4c9f-847d-d6aaacb15be8 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:009a4062-c638-4c9f-847d-d6aaacb15be8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:40,078 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:18:40,079 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:40,079 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:40,079 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:40,079 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:40,084 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:40,084 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:40,089 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:40,090 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:40,091 [254] [DEBUG] [app] Ending request: urn:request:c06e996b-86c0-41b0-a3fa-c479580dd0a6 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c06e996b-86c0-41b0-a3fa-c479580dd0a6', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:40,092 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:40,092 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.027 118 0.026) gunicorn-web stdout | 2025-11-04 09:18:40,122 [254] [DEBUG] [app] Starting request: urn:request:fb7ef3a6-66b9-4896-967f-67b2cdff8008 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:18:40,124 [260] [DEBUG] [app] Starting request: urn:request:77f0e756-d6cc-439a-8483-6961a1b77469 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:40,124 [260] [DEBUG] [app] Ending request: urn:request:77f0e756-d6cc-439a-8483-6961a1b77469 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:77f0e756-d6cc-439a-8483-6961a1b77469', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:40,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:40,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:18:40,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:40,128 [257] [DEBUG] [app] Starting request: urn:request:b620af96-0c4a-43a5-806d-5253e8d4b63c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:40,128 [257] [DEBUG] [app] Ending request: urn:request:b620af96-0c4a-43a5-806d-5253e8d4b63c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b620af96-0c4a-43a5-806d-5253e8d4b63c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:40,128 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:18:40,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:40,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:40,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:40,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:40,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:40,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:40,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:40,140 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:40,140 [254] [DEBUG] [app] Ending request: urn:request:fb7ef3a6-66b9-4896-967f-67b2cdff8008 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:fb7ef3a6-66b9-4896-967f-67b2cdff8008', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:40,140 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:40,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) repositorygcworker stdout | 2025-11-04 09:18:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:18:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:06.265965+00:00 (in 26.084746 seconds) repositorygcworker stdout | 2025-11-04 09:18:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:40 GMT)" (scheduled at 2025-11-04 09:18:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:18:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:18:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 40, 181855), True, datetime.datetime(2025, 11, 4, 9, 18, 40, 181855), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:18:40,190 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:18:40,190 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:18:40,190 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:18:40,297 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:18:40,315 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:18:41,119 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:18:41,389 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:18:41,706 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:42,376 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:42,392 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:18:42,392 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:18:42,805 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:43,206 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:18:43,606 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:18:44,407 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:44,889 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:44,904 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:44,916 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:18:44,931 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:18:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:18:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:14.980431+00:00 (in 29.999501 seconds) buildlogsarchiver stdout | 2025-11-04 09:18:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:14 GMT)" (scheduled at 2025-11-04 09:18:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:18:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 18, 44, 981263), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:18:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:18:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:18:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:18:45,057 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,406 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,483 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,527 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,560 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,575 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,580 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,582 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,594 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:18:46,593 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:49.676299+00:00 (in 1.995808 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:47 GMT)" (scheduled at 2025-11-04 09:18:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:18:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:57.766253+00:00 (in 9.999575 seconds) notificationworker stdout | 2025-11-04 09:18:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:57 GMT)" (scheduled at 2025-11-04 09:18:47.766253+00:00) notificationworker stdout | 2025-11-04 09:18:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 47, 767866), True, datetime.datetime(2025, 11, 4, 9, 18, 47, 767866), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:47,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:47,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:47,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:18:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:18:59.676299+00:00 (in 9.999486 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:59 GMT)" (scheduled at 2025-11-04 09:18:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 49, 677223), True, datetime.datetime(2025, 11, 4, 9, 18, 49, 677223), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:49,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:49,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:49,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:18:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:18:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:18:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:01.707296+00:00 (in 12.000322 seconds) securityworker stdout | 2025-11-04 09:18:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:19 GMT)" (scheduled at 2025-11-04 09:18:49.706478+00:00) securityworker stdout | 2025-11-04 09:18:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:18:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:18:49,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:18:49,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:18:49,718 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:18:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 35, 45]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:18:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:18:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:18:49,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 45, 49]) securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-35 securityworker stdout | 2025-11-04 09:18:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:18:49,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 17, 27]) securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 17-27 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 17-27 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 17-27 securityworker stdout | 2025-11-04 09:18:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-17 securityworker stdout | 2025-11-04 09:18:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 7 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 7 securityworker stdout | 2025-11-04 09:18:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 2, 12]) securityworker stdout | 2025-11-04 09:18:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Right range 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-17 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-17 securityworker stdout | 2025-11-04 09:18:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:18:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 12, 22]) securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Merging with block 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-27 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Left range 2-27 securityworker stdout | 2025-11-04 09:18:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 27-35 securityworker stdout | 2025-11-04 09:18:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 27-35 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:18:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 27, 37]) securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-27 securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-27 securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 27-37 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 27-37 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:18:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:18:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 3, 13]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:18:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-49 securityworker stdout | 2025-11-04 09:18:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:18:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 32, 42]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stdout | 2025-11-04 09:18:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-32 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-32 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:18:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 22 securityworker stdout | 2025-11-04 09:18:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 16, 26]) securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 16-26 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 16-26 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 16-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 16-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-16 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 13-16 securityworker stdout | 2025-11-04 09:18:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 13 securityworker stdout | 2025-11-04 09:18:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 13, 23]) securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-26 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Left range 3-26 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stdout | 2025-11-04 09:18:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-32 securityworker stdout | 2025-11-04 09:18:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-13 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 16-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 3-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 26-32 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 26 securityworker stdout | 2025-11-04 09:18:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 26, 36]) securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 26-36 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 26-36 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-26 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-26 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-42 securityworker stdout | 2025-11-04 09:18:49,746 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 3-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 3-26 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [util.migrate.allocator] Right range 3-42 securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 3-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:49,747 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 1, 11]) securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-42 securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stdout | 2025-11-04 09:18:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:18:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:18:49,749 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 42, 49]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:18:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:18:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 7, 17]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stdout | 2025-11-04 09:18:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:18:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 1, 11]) securityworker stdout | 2025-11-04 09:18:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-49 securityworker stdout | 2025-11-04 09:18:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:18:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 31, 41]) securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-31 securityworker stdout | 2025-11-04 09:18:49,757 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-31 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:18:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 21, 31]) securityworker stdout | 2025-11-04 09:18:49,758 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:18:49,758 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-41 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-41 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Left range 21-41 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:18:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Left range 21-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:18:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 41, 49]) securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-41 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-41 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 17-21 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 21-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 21-41 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 21 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-21 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-21 securityworker stdout | 2025-11-04 09:18:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-21 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:18:49,763 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 13, 49, 712037), 17, 27]) securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:18:49,764 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:18:49,764 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stderr | 2025-11-04 09:18:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:18:51,224 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:00.086435+00:00 (in 7.925997 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:52 GMT)" (scheduled at 2025-11-04 09:18:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,169 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:18:52,169 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:18:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:18:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:33.561829+00:00 (in 40.000851 seconds) namespacegcworker stdout | 2025-11-04 09:18:53,561 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:53 GMT)" (scheduled at 2025-11-04 09:18:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:18:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:18:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 53, 561390), True, datetime.datetime(2025, 11, 4, 9, 18, 53, 561390), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:18:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:18:53,570 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:18:53,571 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:19:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:18:55,066 [258] [DEBUG] [app] Starting request: urn:request:d534a59b-e0d9-497e-97a9-1b2f0cf0aa9d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:55,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:55,069 [260] [DEBUG] [app] Starting request: urn:request:c38e44dc-49a5-4259-904d-10f685f3a7ca (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:55,070 [260] [DEBUG] [app] Ending request: urn:request:c38e44dc-49a5-4259-904d-10f685f3a7ca (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c38e44dc-49a5-4259-904d-10f685f3a7ca', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:18:55,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:18:55,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:55,072 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:55,073 [254] [DEBUG] [app] Starting request: urn:request:b1565bb6-6c29-41de-b42f-12bb80d7e1c0 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:55,074 [254] [DEBUG] [app] Ending request: urn:request:b1565bb6-6c29-41de-b42f-12bb80d7e1c0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:b1565bb6-6c29-41de-b42f-12bb80d7e1c0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:55,074 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:18:55,074 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:55,075 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:55,075 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:55,075 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:55,081 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:55,081 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:55,087 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:55,089 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:55,090 [258] [DEBUG] [app] Ending request: urn:request:d534a59b-e0d9-497e-97a9-1b2f0cf0aa9d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d534a59b-e0d9-497e-97a9-1b2f0cf0aa9d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:55,090 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:55,090 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.025 118 0.026) gunicorn-web stdout | 2025-11-04 09:18:55,123 [261] [DEBUG] [app] Starting request: urn:request:2a564836-9010-4f5e-baba-2f08eef0ee6d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:18:55,124 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:18:55,126 [260] [DEBUG] [app] Starting request: urn:request:694fcd71-8649-4eb1-8a17-1df8d62f5a47 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:18:55,127 [260] [DEBUG] [app] Ending request: urn:request:694fcd71-8649-4eb1-8a17-1df8d62f5a47 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:694fcd71-8649-4eb1-8a17-1df8d62f5a47', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:18:55,127 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:18:55,127 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:55,128 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:18:55,130 [254] [DEBUG] [app] Starting request: urn:request:df87aca3-b7bc-42ef-9c24-259f8bd91b00 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:18:55,131 [254] [DEBUG] [app] Ending request: urn:request:df87aca3-b7bc-42ef-9c24-259f8bd91b00 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:df87aca3-b7bc-42ef-9c24-259f8bd91b00', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:18:55,131 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:18:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:18:55,132 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:18:55,132 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:18:55,132 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:18:55,132 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:18:55,138 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:18:55,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:18:55,143 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:18:55,144 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:18:55,145 [261] [DEBUG] [app] Ending request: urn:request:2a564836-9010-4f5e-baba-2f08eef0ee6d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2a564836-9010-4f5e-baba-2f08eef0ee6d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:18:55,145 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:18:55,145 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:18:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:18:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) gcworker stdout | 2025-11-04 09:18:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:18:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:07.663502+00:00 (in 12.001351 seconds) gcworker stdout | 2025-11-04 09:18:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:25 GMT)" (scheduled at 2025-11-04 09:18:55.661677+00:00) gcworker stdout | 2025-11-04 09:18:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038335662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:18:55,674 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:18:55,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:18:55,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:18:55,923 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:18:56,331 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:18:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:18:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:07.766253+00:00 (in 9.999554 seconds) notificationworker stdout | 2025-11-04 09:18:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:07 GMT)" (scheduled at 2025-11-04 09:18:57.766253+00:00) notificationworker stdout | 2025-11-04 09:18:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:18:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 57, 767038), True, datetime.datetime(2025, 11, 4, 9, 18, 57, 767038), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:18:57,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:18:57,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:18:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:18:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:18:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:09.676299+00:00 (in 9.999471 seconds) proxycacheblobworker stdout | 2025-11-04 09:18:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:09 GMT)" (scheduled at 2025-11-04 09:18:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:18:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:18:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 18, 59, 677205), True, datetime.datetime(2025, 11, 4, 9, 18, 59, 677205), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:18:59,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:18:59,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:18:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:52.159714+00:00 (in 52.072780 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:00 GMT)" (scheduled at 2025-11-04 09:19:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:19:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:19:00,135 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:19:00,233 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:00,520 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:19:01,345 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:19:01,532 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:19.706478+00:00 (in 17.998594 seconds) securityworker stdout | 2025-11-04 09:19:01,708 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:31 GMT)" (scheduled at 2025-11-04 09:19:01.707296+00:00) securityworker stdout | 2025-11-04 09:19:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:01,714 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:01,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:01,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:01,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:01,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:01,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:01,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:01,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:01,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 1, 714032), 1, 49]) securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:01,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:01,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 1, 714032), 1, 49]) securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:01,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:01,731 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:01,731 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:19:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:19:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:32.081579+00:00 (in 29.999528 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:19:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:32 GMT)" (scheduled at 2025-11-04 09:19:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:19:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:19:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:19:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:19:02,717 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:19:04,330 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:19:04,423 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:19:05,520 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:19:06,012 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:40.180734+00:00 (in 33.913807 seconds) repositorygcworker stdout | 2025-11-04 09:19:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:06 GMT)" (scheduled at 2025-11-04 09:19:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:19:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:19:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:19:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:19:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:33.568037+00:00 (in 26.998024 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:19:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:06 GMT)" (scheduled at 2025-11-04 09:19:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:19:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:19:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:19:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:19:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:37.077358+00:00 (in 29.999469 seconds) autopruneworker stdout | 2025-11-04 09:19:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:37 GMT)" (scheduled at 2025-11-04 09:19:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:19:07,085 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244347084, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:19:07,088 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:19:07,088 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:19:07,089 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:19:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:07,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:25.661677+00:00 (in 17.997697 seconds) gcworker stdout | 2025-11-04 09:19:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:37 GMT)" (scheduled at 2025-11-04 09:19:07.663502+00:00) gcworker stdout | 2025-11-04 09:19:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:19:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247647671, None, 1, 0]) gcworker stdout | 2025-11-04 09:19:07,674 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:07,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:17.766253+00:00 (in 9.999529 seconds) notificationworker stdout | 2025-11-04 09:19:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:17 GMT)" (scheduled at 2025-11-04 09:19:07.766253+00:00) notificationworker stdout | 2025-11-04 09:19:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 7, 767070), True, datetime.datetime(2025, 11, 4, 9, 19, 7, 767070), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:19:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:19:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:07.770322+00:00 (in 59.999508 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:19:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:07 GMT)" (scheduled at 2025-11-04 09:19:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:19:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:19:07,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:19:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:19:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:19:08,526 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:19:09,627 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:19.676299+00:00 (in 9.999474 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:09,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:19 GMT)" (scheduled at 2025-11-04 09:19:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 9, 677252), True, datetime.datetime(2025, 11, 4, 9, 19, 9, 677252), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:09,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:09,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:09,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:19:09,908 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:10,066 [254] [DEBUG] [app] Starting request: urn:request:244167a9-ad6f-427e-9ee7-d1db2cb0e4d2 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:19:10,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:19:10,070 [250] [DEBUG] [app] Starting request: urn:request:f32856f1-90bd-4247-86fa-615d35b6dcee (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:10,070 [250] [DEBUG] [app] Ending request: urn:request:f32856f1-90bd-4247-86fa-615d35b6dcee (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f32856f1-90bd-4247-86fa-615d35b6dcee', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:10,071 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:10,071 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:19:10,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:10,074 [254] [DEBUG] [app] Starting request: urn:request:1ac3211e-8bf5-4c94-930a-da481f9ee4f0 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:10,074 [254] [DEBUG] [app] Ending request: urn:request:1ac3211e-8bf5-4c94-930a-da481f9ee4f0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:1ac3211e-8bf5-4c94-930a-da481f9ee4f0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:10,075 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:10,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:10,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:10,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:10,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:10,081 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:10,081 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:10,085 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:10,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:10,088 [254] [DEBUG] [app] Ending request: urn:request:244167a9-ad6f-427e-9ee7-d1db2cb0e4d2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:244167a9-ad6f-427e-9ee7-d1db2cb0e4d2', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:10,089 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:10,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:19:10,123 [261] [DEBUG] [app] Starting request: urn:request:4455e094-b548-4f93-82d8-57546ad77cd4 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:19:10,130 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:19:10,132 [259] [DEBUG] [app] Starting request: urn:request:dbbaa967-49e0-48db-8e04-e8effb860e47 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:10,133 [259] [DEBUG] [app] Ending request: urn:request:dbbaa967-49e0-48db-8e04-e8effb860e47 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:dbbaa967-49e0-48db-8e04-e8effb860e47', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:10,133 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:10,133 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:10,135 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:10,136 [261] [DEBUG] [app] Starting request: urn:request:d3a53576-b6a3-424c-ba6f-8771ec343a0d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:10,136 [261] [DEBUG] [app] Ending request: urn:request:d3a53576-b6a3-424c-ba6f-8771ec343a0d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d3a53576-b6a3-424c-ba6f-8771ec343a0d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:10,137 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:10,137 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:10,138 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:10,138 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:10,138 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:10,143 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:10,143 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:10,149 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:10,150 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:10,151 [261] [DEBUG] [app] Ending request: urn:request:4455e094-b548-4f93-82d8-57546ad77cd4 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4455e094-b548-4f93-82d8-57546ad77cd4', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:10,151 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:10,151 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.029 118 0.030) securityworker stdout | 2025-11-04 09:19:10,313 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:19:10,332 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:19:11,135 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:19:11,406 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:19:11,723 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:12,395 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:12,410 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:12,412 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:19:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:19:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:38.480592+00:00 (in 25.997976 seconds) exportactionlogsworker stdout | 2025-11-04 09:19:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:12 GMT)" (scheduled at 2025-11-04 09:19:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:19:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:19:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:19:12,822 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:13,223 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:13,624 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:19:14,424 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:14,909 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:14,925 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:14,935 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:14,949 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:19:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:19:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:44.980431+00:00 (in 29.999519 seconds) buildlogsarchiver stdout | 2025-11-04 09:19:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:44 GMT)" (scheduled at 2025-11-04 09:19:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:19:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 19, 14, 981504), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:14,991 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:19:14,991 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:19:14,991 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:15,076 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,427 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,508 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,548 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,580 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,597 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,602 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,604 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,612 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:16,614 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:27.766253+00:00 (in 9.999309 seconds) notificationworker stdout | 2025-11-04 09:19:17,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:27 GMT)" (scheduled at 2025-11-04 09:19:17.766253+00:00) notificationworker stdout | 2025-11-04 09:19:17,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 17, 767877), True, datetime.datetime(2025, 11, 4, 9, 19, 17, 767877), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:17,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:17,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:17,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:19:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:29.676299+00:00 (in 9.999483 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:19,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:29 GMT)" (scheduled at 2025-11-04 09:19:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 19, 677229), True, datetime.datetime(2025, 11, 4, 9, 19, 19, 677229), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:19,686 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:19,686 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:19,686 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:19:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:31.707296+00:00 (in 12.000356 seconds) securityworker stdout | 2025-11-04 09:19:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:49 GMT)" (scheduled at 2025-11-04 09:19:19.706478+00:00) securityworker stdout | 2025-11-04 09:19:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:19,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:19,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:19,719 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:19:19,719 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:19:21,240 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:25,066 [254] [DEBUG] [app] Starting request: urn:request:c2616d8c-8dba-4ef2-8441-387967216c6e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:19:25,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:19:25,076 [250] [DEBUG] [app] Starting request: urn:request:ecae8275-0751-4aaa-9196-98c4672924ba (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:25,076 [250] [DEBUG] [app] Ending request: urn:request:ecae8275-0751-4aaa-9196-98c4672924ba (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ecae8275-0751-4aaa-9196-98c4672924ba', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:25,077 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:25,077 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:25,078 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:25,080 [261] [DEBUG] [app] Starting request: urn:request:c84ca313-9553-4588-ac4d-a7fc1daf5744 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:25,081 [261] [DEBUG] [app] Ending request: urn:request:c84ca313-9553-4588-ac4d-a7fc1daf5744 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c84ca313-9553-4588-ac4d-a7fc1daf5744', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:25,082 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:25,082 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:25,082 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:25,082 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:25,082 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:25,091 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:25,091 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:25,097 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:25,099 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:25,100 [254] [DEBUG] [app] Ending request: urn:request:c2616d8c-8dba-4ef2-8441-387967216c6e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c2616d8c-8dba-4ef2-8441-387967216c6e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:25,100 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.036 118 0.036) gunicorn-web stdout | 2025-11-04 09:19:25,100 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:19:25,122 [254] [DEBUG] [app] Starting request: urn:request:c38350ea-943b-46a4-8f4c-641bdfa42256 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:19:25,124 [260] [DEBUG] [app] Starting request: urn:request:7b9567ad-94cf-4917-9c58-7d6734218357 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:25,125 [260] [DEBUG] [app] Ending request: urn:request:7b9567ad-94cf-4917-9c58-7d6734218357 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7b9567ad-94cf-4917-9c58-7d6734218357', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:19:25,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:25,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:25,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:25,128 [261] [DEBUG] [app] Starting request: urn:request:4a9acbdc-0fdd-4969-bd70-e36887c4da67 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:25,129 [261] [DEBUG] [app] Ending request: urn:request:4a9acbdc-0fdd-4969-bd70-e36887c4da67 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4a9acbdc-0fdd-4969-bd70-e36887c4da67', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:25,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:25,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:25,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:25,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:25,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:25,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:25,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:25,139 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:25,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:25,142 [254] [DEBUG] [app] Ending request: urn:request:c38350ea-943b-46a4-8f4c-641bdfa42256 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c38350ea-943b-46a4-8f4c-641bdfa42256', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:25,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:25,142 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gcworker stdout | 2025-11-04 09:19:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:37.663502+00:00 (in 12.001351 seconds) gcworker stdout | 2025-11-04 09:19:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:55 GMT)" (scheduled at 2025-11-04 09:19:25.661677+00:00) gcworker stdout | 2025-11-04 09:19:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038365662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:19:25,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:19:25,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:25,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:19:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:25,939 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:19:26,348 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:27.768179+00:00 (in 0.001424 seconds) notificationworker stdout | 2025-11-04 09:19:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:37 GMT)" (scheduled at 2025-11-04 09:19:27.766253+00:00) notificationworker stdout | 2025-11-04 09:19:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 27, 767233), True, datetime.datetime(2025, 11, 4, 9, 19, 27, 767233), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:37.766253+00:00 (in 9.997287 seconds) notificationworker stdout | 2025-11-04 09:19:27,769 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:27 GMT)" (scheduled at 2025-11-04 09:19:27.768179+00:00) notificationworker stdout | 2025-11-04 09:19:27,769 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:19:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:39.676299+00:00 (in 9.999447 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:29,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:39 GMT)" (scheduled at 2025-11-04 09:19:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:29,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 29, 677160), True, datetime.datetime(2025, 11, 4, 9, 19, 29, 677160), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:29,687 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:29,687 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:29,687 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:19:30,151 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:19:30,250 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:19:30,537 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:19:31,365 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:19:31,549 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:19:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:49.706478+00:00 (in 17.998657 seconds) securityworker stdout | 2025-11-04 09:19:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:01 GMT)" (scheduled at 2025-11-04 09:19:31.707296+00:00) securityworker stdout | 2025-11-04 09:19:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:31,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:31,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:31,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:31,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:31,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:31,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:31,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 31, 713686), 1, 49]) securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:31,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:31,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 31, 713686), 1, 49]) securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:31,730 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:19:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:31,730 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:19:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:19:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:02.081579+00:00 (in 29.999539 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:19:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:02 GMT)" (scheduled at 2025-11-04 09:19:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:19:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:19:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:19:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:19:32,735 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:19:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:19:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:53.560131+00:00 (in 19.997809 seconds) namespacegcworker stdout | 2025-11-04 09:19:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" (scheduled at 2025-11-04 09:19:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:19:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:19:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:19:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:19:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:06.569529+00:00 (in 33.000915 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:19:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" (scheduled at 2025-11-04 09:19:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:19:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:19:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 33, 568935), True, datetime.datetime(2025, 11, 4, 9, 19, 33, 568935), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:19:33,578 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:19:33,578 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:19:33,578 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:19:34,347 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:19:34,438 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:19:35,537 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:19:36,029 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:19:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:19:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:07.077358+00:00 (in 29.999414 seconds) autopruneworker stdout | 2025-11-04 09:19:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:07 GMT)" (scheduled at 2025-11-04 09:19:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:19:37,084 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244377082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:19:37,087 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:19:37,087 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:19:37,087 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:19:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:37,664 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:55.661677+00:00 (in 17.997641 seconds) gcworker stdout | 2025-11-04 09:19:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:07 GMT)" (scheduled at 2025-11-04 09:19:37.663502+00:00) gcworker stdout | 2025-11-04 09:19:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:19:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247677671, None, 1, 0]) gcworker stdout | 2025-11-04 09:19:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:47.766253+00:00 (in 9.999537 seconds) notificationworker stdout | 2025-11-04 09:19:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:47 GMT)" (scheduled at 2025-11-04 09:19:37.766253+00:00) notificationworker stdout | 2025-11-04 09:19:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 37, 767053), True, datetime.datetime(2025, 11, 4, 9, 19, 37, 767053), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:19:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:19:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:12.482065+00:00 (in 34.000950 seconds) exportactionlogsworker stdout | 2025-11-04 09:19:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:38 GMT)" (scheduled at 2025-11-04 09:19:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:19:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:19:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 38, 481517), True, datetime.datetime(2025, 11, 4, 9, 19, 38, 481517), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:19:38,490 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:19:38,490 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:19:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:19:38,542 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:19:39,660 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:44.679698+00:00 (in 5.002960 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:49 GMT)" (scheduled at 2025-11-04 09:19:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 39, 677150), True, datetime.datetime(2025, 11, 4, 9, 19, 39, 677150), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:39,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:39,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:39,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:49 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:19:39,925 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:40,065 [254] [DEBUG] [app] Starting request: urn:request:4794c1df-d530-4645-93c8-e1c2e8879256 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:19:40,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:19:40,068 [250] [DEBUG] [app] Starting request: urn:request:6c09222d-90fc-45dc-b1c9-11d9c5f044e9 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:40,069 [250] [DEBUG] [app] Ending request: urn:request:6c09222d-90fc-45dc-b1c9-11d9c5f044e9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6c09222d-90fc-45dc-b1c9-11d9c5f044e9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:40,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:40,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:40,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:40,072 [258] [DEBUG] [app] Starting request: urn:request:15052ce4-b382-4cd9-b2dc-301ce4def235 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:40,073 [258] [DEBUG] [app] Ending request: urn:request:15052ce4-b382-4cd9-b2dc-301ce4def235 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:15052ce4-b382-4cd9-b2dc-301ce4def235', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:40,073 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:40,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:40,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:40,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:40,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:40,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:40,084 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:40,085 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:40,086 [254] [DEBUG] [app] Ending request: urn:request:4794c1df-d530-4645-93c8-e1c2e8879256 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:4794c1df-d530-4645-93c8-e1c2e8879256', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:40,087 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:40,087 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) gunicorn-web stdout | 2025-11-04 09:19:40,122 [254] [DEBUG] [app] Starting request: urn:request:5855ffa3-737f-4369-a270-141ddd8968d9 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:19:40,124 [250] [DEBUG] [app] Starting request: urn:request:3f83902a-7d26-429e-81be-1601d242ddd0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:40,125 [250] [DEBUG] [app] Ending request: urn:request:3f83902a-7d26-429e-81be-1601d242ddd0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3f83902a-7d26-429e-81be-1601d242ddd0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:40,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:40,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:40,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:40,128 [258] [DEBUG] [app] Starting request: urn:request:5ce9d5e6-1c6e-4adb-abd6-2acc5e6c0112 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:40,129 [258] [DEBUG] [app] Ending request: urn:request:5ce9d5e6-1c6e-4adb-abd6-2acc5e6c0112 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5ce9d5e6-1c6e-4adb-abd6-2acc5e6c0112', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:40,129 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:40,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:40,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:40,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:40,131 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:40,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:40,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:40,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:40,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:40,142 [254] [DEBUG] [app] Ending request: urn:request:5855ffa3-737f-4369-a270-141ddd8968d9 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:5855ffa3-737f-4369-a270-141ddd8968d9', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:40,142 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gunicorn-web stdout | 2025-11-04 09:19:40,143 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:19:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:41.259858+00:00 (in 1.078618 seconds) repositorygcworker stdout | 2025-11-04 09:19:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:40 GMT)" (scheduled at 2025-11-04 09:19:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:19:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:19:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 40, 181611), True, datetime.datetime(2025, 11, 4, 9, 19, 40, 181611), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:19:40,190 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:19:40,190 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:19:40,190 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:19:40,333 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:19:40,349 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:19:41,152 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:19:41,260 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:19:41,260 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:06.265965+00:00 (in 25.005493 seconds) repositorygcworker stdout | 2025-11-04 09:19:41,260 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:41 GMT)" (scheduled at 2025-11-04 09:19:41.259858+00:00) repositorygcworker stdout | 2025-11-04 09:19:41,261 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:41,268 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), True, datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), 0, 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:41,270 [90] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), True, datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), 0, 'repositorygc/%', False, datetime.datetime(2025, 11, 4, 9, 19, 41, 260737), 'repositorygc/%']) repositorygcworker stdout | 2025-11-04 09:19:41,271 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:19:41,271 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:41 GMT)" executed successfully blobuploadcleanupworker stdout | 2025-11-04 09:19:41,440 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:19:41,739 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:42,412 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:42,428 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:19:42,431 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:19:42,838 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:43,240 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:19:43,641 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:19:44,442 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:44,679 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:44,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:47.679968+00:00 (in 2.999773 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:44,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:44 GMT)" (scheduled at 2025-11-04 09:19:44.679698+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:44,680 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:44,689 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), True, datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), 0, 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:44,692 [78] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), True, datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), 0, 'proxycacheblob/%', False, datetime.datetime(2025, 11, 4, 9, 19, 44, 680486), 'proxycacheblob/%']) proxycacheblobworker stdout | 2025-11-04 09:19:44,693 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:44,693 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:24:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:44,929 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:44,953 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:44,956 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:19:44,968 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:19:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:19:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:14.980431+00:00 (in 29.999547 seconds) buildlogsarchiver stdout | 2025-11-04 09:19:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:14 GMT)" (scheduled at 2025-11-04 09:19:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:19:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 19, 44, 981143), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:19:44,991 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:19:44,991 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:19:44,991 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:45,095 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,447 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,529 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,568 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,601 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,617 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,624 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,630 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,636 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:19:46,637 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:49.676299+00:00 (in 1.995820 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:47,681 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:47 GMT)" (scheduled at 2025-11-04 09:19:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:19:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:57.766253+00:00 (in 9.999565 seconds) notificationworker stdout | 2025-11-04 09:19:47,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:57 GMT)" (scheduled at 2025-11-04 09:19:47.766253+00:00) notificationworker stdout | 2025-11-04 09:19:47,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:47,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 47, 768020), True, datetime.datetime(2025, 11, 4, 9, 19, 47, 768020), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:47,777 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:47,777 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:47,777 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:19:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:19:59.676299+00:00 (in 9.999440 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:59 GMT)" (scheduled at 2025-11-04 09:19:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:49,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 49, 677308), True, datetime.datetime(2025, 11, 4, 9, 19, 49, 677308), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:49,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:49,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:49,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:19:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:19:49,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:19:49,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:01.707296+00:00 (in 11.999987 seconds) securityworker stdout | 2025-11-04 09:19:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:19 GMT)" (scheduled at 2025-11-04 09:19:49.706478+00:00) securityworker stdout | 2025-11-04 09:19:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:19:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:19:49,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:19:49,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:49,717 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:19:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 2, 12]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,722 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:19:49,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-49 securityworker stdout | 2025-11-04 09:19:49,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 31, 41]) securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:19:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:19:49,724 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:19:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:19:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-31 securityworker stdout | 2025-11-04 09:19:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:19:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 14, 24]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:19:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 4 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-31 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-31 securityworker stdout | 2025-11-04 09:19:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:19:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 24, 34]) securityworker stdout | 2025-11-04 09:19:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-41 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Right range 14-41 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-14 securityworker stdout | 2025-11-04 09:19:49,728 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 14-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-14 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:19:49,729 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 12, 22]) securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with block 14-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 14-41 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-41 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,730 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,731 [92] [DEBUG] [util.migrate.allocator] Left range 2-41 securityworker stdout | 2025-11-04 09:19:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:19:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:19:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 41, 49]) securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-41 securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-41 securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stdout | 2025-11-04 09:19:49,732 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:19:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:19:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 1, 11]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-49 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 31, 41]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:19:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Left range 31-41 securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:19:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:19:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 41, 49]) securityworker stdout | 2025-11-04 09:19:49,741 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:19:49,741 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 31-41 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Already merged with block 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 31-41 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 31 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 11-31 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-31 securityworker stdout | 2025-11-04 09:19:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:19:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 13, 23]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 11-31 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 13-23 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-13 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 13-23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 11-13 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:19:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:19:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 11, 21]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-23 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 23-31 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 23-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-31 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:19:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:19:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 23, 33]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 23 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 33 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 33-23 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 33 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 33-23 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:19:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 20, 30]) securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-20 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-20 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 10 securityworker stdout | 2025-11-04 09:19:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 10 securityworker stdout | 2025-11-04 09:19:49,754 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 2, 12]) securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-20 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-20 securityworker stdout | 2025-11-04 09:19:49,755 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:19:49,756 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 12, 22]) securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Merging with block 20-30 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 20-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-30 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,758 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,758 [92] [DEBUG] [util.migrate.allocator] Left range 2-30 securityworker stdout | 2025-11-04 09:19:49,758 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-49 securityworker stdout | 2025-11-04 09:19:49,758 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:19:49,759 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 33, 43]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:19:49,760 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:19:49,760 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:19:49,760 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Left range 2-30 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-33 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-33 securityworker stdout | 2025-11-04 09:19:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stdout | 2025-11-04 09:19:49,762 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 30, 40]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-30 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-30 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-30 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:19:49,763 [92] [DEBUG] [util.migrate.allocator] Left range 2-43 securityworker stdout | 2025-11-04 09:19:49,764 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stdout | 2025-11-04 09:19:49,764 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stdout | 2025-11-04 09:19:49,765 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 43, 49]) securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-43 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-43 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:19:49,766 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-43 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:19:49,767 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 14, 49, 711187), 1, 11]) securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:49,769 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stderr | 2025-11-04 09:19:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:19:49,769 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:19:51,256 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:00.086435+00:00 (in 7.925853 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:52 GMT)" (scheduled at 2025-11-04 09:19:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,168 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,168 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:19:52,168 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:19:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:19:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:33.561829+00:00 (in 40.001075 seconds) namespacegcworker stdout | 2025-11-04 09:19:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:53 GMT)" (scheduled at 2025-11-04 09:19:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:19:53,561 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:19:53,562 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 53, 561100), True, datetime.datetime(2025, 11, 4, 9, 19, 53, 561100), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:19:53,570 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:19:53,570 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:19:53,570 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:20:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:19:55,065 [254] [DEBUG] [app] Starting request: urn:request:8e33c442-e198-4eb4-aa82-60b9ea314370 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:19:55,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:19:55,069 [250] [DEBUG] [app] Starting request: urn:request:e83e2ee4-667f-4d9e-a7cf-ede4444b53c5 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:55,069 [250] [DEBUG] [app] Ending request: urn:request:e83e2ee4-667f-4d9e-a7cf-ede4444b53c5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e83e2ee4-667f-4d9e-a7cf-ede4444b53c5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:55,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:55,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:55,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:55,073 [261] [DEBUG] [app] Starting request: urn:request:d20bba03-8c82-4ba2-997c-33500732b94e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:55,073 [261] [DEBUG] [app] Ending request: urn:request:d20bba03-8c82-4ba2-997c-33500732b94e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d20bba03-8c82-4ba2-997c-33500732b94e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:19:55,074 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:55,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:55,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:55,075 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:55,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:19:55,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:55,080 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:55,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:55,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:55,088 [254] [DEBUG] [app] Ending request: urn:request:8e33c442-e198-4eb4-aa82-60b9ea314370 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8e33c442-e198-4eb4-aa82-60b9ea314370', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:55,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:55,089 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:19:55,122 [254] [DEBUG] [app] Starting request: urn:request:ad3f401e-3a4e-42ee-ad7f-215ce7bf08bb (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:19:55,124 [260] [DEBUG] [app] Starting request: urn:request:8df7e523-14ca-4fc5-9ec2-1b8440e582e5 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:19:55,125 [260] [DEBUG] [app] Ending request: urn:request:8df7e523-14ca-4fc5-9ec2-1b8440e582e5 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:8df7e523-14ca-4fc5-9ec2-1b8440e582e5', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:19:55,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:55,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:19:55,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:19:55,129 [261] [DEBUG] [app] Starting request: urn:request:2850c4ed-0d3d-4db1-a230-bbe84d7b442e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:19:55,129 [261] [DEBUG] [app] Ending request: urn:request:2850c4ed-0d3d-4db1-a230-bbe84d7b442e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2850c4ed-0d3d-4db1-a230-bbe84d7b442e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:19:55,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:19:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:19:55,130 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:19:55,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:19:55,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:19:55,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:19:55,135 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:19:55,135 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:19:55,140 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:19:55,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:19:55,142 [254] [DEBUG] [app] Ending request: urn:request:ad3f401e-3a4e-42ee-ad7f-215ce7bf08bb (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ad3f401e-3a4e-42ee-ad7f-215ce7bf08bb', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:19:55,142 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:19:55,142 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:19:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:19:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gcworker stdout | 2025-11-04 09:19:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:19:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:07.663502+00:00 (in 12.001341 seconds) gcworker stdout | 2025-11-04 09:19:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:25 GMT)" (scheduled at 2025-11-04 09:19:55.661677+00:00) gcworker stdout | 2025-11-04 09:19:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038395662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:19:55,673 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:19:55,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:19:55,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:19:55,957 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:19:56,366 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:19:57,767 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:19:57,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:07.766253+00:00 (in 9.998966 seconds) notificationworker stdout | 2025-11-04 09:19:57,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:07 GMT)" (scheduled at 2025-11-04 09:19:57.766253+00:00) notificationworker stdout | 2025-11-04 09:19:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:19:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 57, 767724), True, datetime.datetime(2025, 11, 4, 9, 19, 57, 767724), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:19:57,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:19:57,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:19:57,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:19:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:19:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:09.676299+00:00 (in 9.999475 seconds) proxycacheblobworker stdout | 2025-11-04 09:19:59,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:09 GMT)" (scheduled at 2025-11-04 09:19:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:19:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:19:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 19, 59, 677181), True, datetime.datetime(2025, 11, 4, 9, 19, 59, 677181), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:19:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:19:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:19:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:52.159714+00:00 (in 52.072812 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:00 GMT)" (scheduled at 2025-11-04 09:20:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:20:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:20:00,164 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:20:00,262 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:00,549 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:20:01,384 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:20:01,562 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:19.706478+00:00 (in 17.998690 seconds) securityworker stdout | 2025-11-04 09:20:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:31 GMT)" (scheduled at 2025-11-04 09:20:01.707296+00:00) securityworker stdout | 2025-11-04 09:20:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:01,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:01,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:01,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:01,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:01,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 1, 713466), 1, 49]) securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 1, 713466), 1, 49]) securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:01,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:20:01,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:01,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:01,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:20:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:20:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:32.081579+00:00 (in 29.999568 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:20:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:32 GMT)" (scheduled at 2025-11-04 09:20:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:20:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:20:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:20:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:20:02,747 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:20:04,360 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:20:04,450 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:20:05,549 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:20:06,043 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:20:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:20:06,267 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:40.180734+00:00 (in 33.913648 seconds) repositorygcworker stdout | 2025-11-04 09:20:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:06 GMT)" (scheduled at 2025-11-04 09:20:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:20:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:20:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:20:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:20:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:33.568037+00:00 (in 26.998067 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:20:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:06 GMT)" (scheduled at 2025-11-04 09:20:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:20:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:20:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:20:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:20:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:37.077358+00:00 (in 29.999515 seconds) autopruneworker stdout | 2025-11-04 09:20:07,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:37 GMT)" (scheduled at 2025-11-04 09:20:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:20:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244407082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:20:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:20:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:20:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:20:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:25.661677+00:00 (in 17.997730 seconds) gcworker stdout | 2025-11-04 09:20:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:37 GMT)" (scheduled at 2025-11-04 09:20:07.663502+00:00) gcworker stdout | 2025-11-04 09:20:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:20:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247707670, None, 1, 0]) gcworker stdout | 2025-11-04 09:20:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:17.766253+00:00 (in 9.999601 seconds) notificationworker stdout | 2025-11-04 09:20:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:17 GMT)" (scheduled at 2025-11-04 09:20:07.766253+00:00) notificationworker stdout | 2025-11-04 09:20:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 7, 766971), True, datetime.datetime(2025, 11, 4, 9, 20, 7, 766971), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:20:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:20:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:07.770322+00:00 (in 59.999565 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:20:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:07 GMT)" (scheduled at 2025-11-04 09:20:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:20:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:20:07,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:07,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:07,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:20:07,777 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:20:07,777 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:20:08,555 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: manifestbackfillworker stdout | 2025-11-04 09:20:09,672 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:19.676299+00:00 (in 9.999552 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:19 GMT)" (scheduled at 2025-11-04 09:20:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 9, 677073), True, datetime.datetime(2025, 11, 4, 9, 20, 9, 677073), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:19 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:20:09,938 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:10,067 [254] [DEBUG] [app] Starting request: urn:request:95cbca5e-5fca-44a2-a4d1-59ddffff963e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:10,068 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:10,070 [259] [DEBUG] [app] Starting request: urn:request:a99ab125-ad83-48e1-8a0a-f2f5e4cafb11 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:10,070 [259] [DEBUG] [app] Ending request: urn:request:a99ab125-ad83-48e1-8a0a-f2f5e4cafb11 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:a99ab125-ad83-48e1-8a0a-f2f5e4cafb11', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:10,070 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:10,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:10,072 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:10,075 [261] [DEBUG] [app] Starting request: urn:request:3631c1d2-59ae-463e-aff4-cc83359896de (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:10,076 [261] [DEBUG] [app] Ending request: urn:request:3631c1d2-59ae-463e-aff4-cc83359896de (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3631c1d2-59ae-463e-aff4-cc83359896de', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:10,076 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:10,076 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:20:10,077 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:10,077 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:10,077 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:10,082 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:10,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:10,086 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:10,087 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:10,087 [254] [DEBUG] [app] Ending request: urn:request:95cbca5e-5fca-44a2-a4d1-59ddffff963e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:95cbca5e-5fca-44a2-a4d1-59ddffff963e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:10,088 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:10,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.022) gunicorn-web stdout | 2025-11-04 09:20:10,122 [261] [DEBUG] [app] Starting request: urn:request:3306bff2-c74c-4070-b71b-7920765d2fb3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:10,124 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:10,125 [259] [DEBUG] [app] Starting request: urn:request:7be62a54-11b3-4fe1-bd1c-533a04459ed9 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:10,126 [259] [DEBUG] [app] Ending request: urn:request:7be62a54-11b3-4fe1-bd1c-533a04459ed9 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7be62a54-11b3-4fe1-bd1c-533a04459ed9', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:10,126 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:10,126 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:10,127 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:10,128 [261] [DEBUG] [app] Starting request: urn:request:bc4ce7a4-14b1-42e2-b7e5-75af174f215a (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:10,129 [261] [DEBUG] [app] Ending request: urn:request:bc4ce7a4-14b1-42e2-b7e5-75af174f215a (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:bc4ce7a4-14b1-42e2-b7e5-75af174f215a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:10,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:10,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:10,130 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:10,130 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:10,130 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:10,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:10,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:10,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:10,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:10,139 [261] [DEBUG] [app] Ending request: urn:request:3306bff2-c74c-4070-b71b-7920765d2fb3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3306bff2-c74c-4070-b71b-7920765d2fb3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:10,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:10,139 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.018) securityworker stdout | 2025-11-04 09:20:10,346 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:20:10,361 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:20:11,165 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:20:11,452 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:20:11,752 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:12,427 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:12,442 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:12,446 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:20:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:20:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:38.480592+00:00 (in 25.998062 seconds) exportactionlogsworker stdout | 2025-11-04 09:20:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:12 GMT)" (scheduled at 2025-11-04 09:20:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:20:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:20:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:12 GMT)" executed successfully servicekey stdout | 2025-11-04 09:20:12,851 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:13,253 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:13,653 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:20:14,475 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:14,945 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:14,968 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:14,971 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:20:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:20:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:44.980431+00:00 (in 29.999529 seconds) buildlogsarchiver stdout | 2025-11-04 09:20:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:44 GMT)" (scheduled at 2025-11-04 09:20:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:20:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 20, 14, 981191), False, 50, 1, 0]) gunicorn-web stdout | 2025-11-04 09:20:14,981 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:20:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:20:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:20:14,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:15,111 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,462 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,545 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,582 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,616 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,635 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,639 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,645 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,654 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:16,667 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:27.766253+00:00 (in 9.999572 seconds) notificationworker stdout | 2025-11-04 09:20:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:27 GMT)" (scheduled at 2025-11-04 09:20:17.766253+00:00) notificationworker stdout | 2025-11-04 09:20:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 17, 766959), True, datetime.datetime(2025, 11, 4, 9, 20, 17, 766959), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:20:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:29.676299+00:00 (in 9.999564 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:29 GMT)" (scheduled at 2025-11-04 09:20:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 19, 677062), True, datetime.datetime(2025, 11, 4, 9, 20, 19, 677062), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:20:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:31.707296+00:00 (in 12.000419 seconds) securityworker stdout | 2025-11-04 09:20:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:49 GMT)" (scheduled at 2025-11-04 09:20:19.706478+00:00) securityworker stdout | 2025-11-04 09:20:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:19,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:19,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:20:19,716 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:20:21,268 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:25,065 [257] [DEBUG] [app] Starting request: urn:request:ea583100-59f1-462c-8167-e61f9e7b47e8 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:25,066 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:25,068 [260] [DEBUG] [app] Starting request: urn:request:76543b6e-1d70-4497-a15c-e4ef80c0449f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:25,068 [260] [DEBUG] [app] Ending request: urn:request:76543b6e-1d70-4497-a15c-e4ef80c0449f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:76543b6e-1d70-4497-a15c-e4ef80c0449f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:25,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:25,069 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:25,070 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:25,071 [257] [DEBUG] [app] Starting request: urn:request:d01e06f6-615b-4dd5-9c59-7506ddd9d478 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:25,072 [257] [DEBUG] [app] Ending request: urn:request:d01e06f6-615b-4dd5-9c59-7506ddd9d478 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:d01e06f6-615b-4dd5-9c59-7506ddd9d478', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:20:25,073 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:25,073 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:25,073 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:25,073 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:25,073 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:25,078 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:25,078 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:25,084 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:25,085 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:25,086 [257] [DEBUG] [app] Ending request: urn:request:ea583100-59f1-462c-8167-e61f9e7b47e8 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ea583100-59f1-462c-8167-e61f9e7b47e8', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:25,086 [257] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.022) gunicorn-web stdout | 2025-11-04 09:20:25,087 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:20:25,122 [254] [DEBUG] [app] Starting request: urn:request:a6094df5-a670-4b6f-a46c-f9178749e80e (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:25,123 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:25,124 [260] [DEBUG] [app] Starting request: urn:request:3c237f5e-313a-4d97-93df-c6c52e69d791 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:25,125 [260] [DEBUG] [app] Ending request: urn:request:3c237f5e-313a-4d97-93df-c6c52e69d791 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:3c237f5e-313a-4d97-93df-c6c52e69d791', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:25,126 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:25,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:25,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:25,129 [257] [DEBUG] [app] Starting request: urn:request:48e8d2e2-c7ec-4c0a-b2e6-37432b669aba (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:25,129 [257] [DEBUG] [app] Ending request: urn:request:48e8d2e2-c7ec-4c0a-b2e6-37432b669aba (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:48e8d2e2-c7ec-4c0a-b2e6-37432b669aba', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:25,129 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:25,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:25,130 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:25,130 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:25,130 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:25,134 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:25,134 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:25,138 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:25,139 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:25,140 [254] [DEBUG] [app] Ending request: urn:request:a6094df5-a670-4b6f-a46c-f9178749e80e (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a6094df5-a670-4b6f-a46c-f9178749e80e', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:25,140 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:25,141 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gcworker stdout | 2025-11-04 09:20:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:37.663502+00:00 (in 12.001401 seconds) gcworker stdout | 2025-11-04 09:20:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:55 GMT)" (scheduled at 2025-11-04 09:20:25.661677+00:00) gcworker stdout | 2025-11-04 09:20:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038425662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:20:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:20:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:20:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:25,987 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:20:26,379 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:20:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:27.768179+00:00 (in 0.001231 seconds) notificationworker stdout | 2025-11-04 09:20:27,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:37 GMT)" (scheduled at 2025-11-04 09:20:27.766253+00:00) notificationworker stdout | 2025-11-04 09:20:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 27, 767295), True, datetime.datetime(2025, 11, 4, 9, 20, 27, 767295), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:27,769 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:37.766253+00:00 (in 9.997261 seconds) notificationworker stdout | 2025-11-04 09:20:27,769 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:27 GMT)" (scheduled at 2025-11-04 09:20:27.768179+00:00) notificationworker stdout | 2025-11-04 09:20:27,769 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:20:27,769 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:39.676299+00:00 (in 9.999524 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:39 GMT)" (scheduled at 2025-11-04 09:20:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:29,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 29, 677137), True, datetime.datetime(2025, 11, 4, 9, 20, 29, 677137), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:20:30,178 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:20:30,275 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:20:30,562 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:20:31,404 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:20:31,574 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:20:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:49.706478+00:00 (in 17.998772 seconds) securityworker stdout | 2025-11-04 09:20:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:01 GMT)" (scheduled at 2025-11-04 09:20:31.707296+00:00) securityworker stdout | 2025-11-04 09:20:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:31,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:31,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:31,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:31,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:31,717 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:31,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:31,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 31, 711520), 1, 49]) securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 31, 711520), 1, 49]) securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:20:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:20:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:20:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:02.081579+00:00 (in 29.999588 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:20:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:02 GMT)" (scheduled at 2025-11-04 09:20:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:20:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:20:32,084 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:20:32,084 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:20:32,759 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:20:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:20:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:53.560131+00:00 (in 19.997839 seconds) namespacegcworker stdout | 2025-11-04 09:20:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" (scheduled at 2025-11-04 09:20:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:20:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:20:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:20:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:20:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:06.569529+00:00 (in 33.001067 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:20:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" (scheduled at 2025-11-04 09:20:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:20:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:20:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 33, 568860), True, datetime.datetime(2025, 11, 4, 9, 20, 33, 568860), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:20:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:20:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:20:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:20:34,373 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:20:34,463 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:20:35,561 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:20:36,055 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:20:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:20:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:07.077358+00:00 (in 29.999563 seconds) autopruneworker stdout | 2025-11-04 09:20:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:07 GMT)" (scheduled at 2025-11-04 09:20:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:20:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244437082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:20:37,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:20:37,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:20:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:20:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:55.661677+00:00 (in 17.997758 seconds) gcworker stdout | 2025-11-04 09:20:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:07 GMT)" (scheduled at 2025-11-04 09:20:37.663502+00:00) gcworker stdout | 2025-11-04 09:20:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:20:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247737670, None, 1, 0]) gcworker stdout | 2025-11-04 09:20:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:37,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:47.766253+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:20:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:47 GMT)" (scheduled at 2025-11-04 09:20:37.766253+00:00) notificationworker stdout | 2025-11-04 09:20:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 37, 766999), True, datetime.datetime(2025, 11, 4, 9, 20, 37, 766999), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:37,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:20:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:20:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:55.481710+00:00 (in 17.000688 seconds) exportactionlogsworker stdout | 2025-11-04 09:20:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:38 GMT)" (scheduled at 2025-11-04 09:20:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:20:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:20:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 38, 481344), True, datetime.datetime(2025, 11, 4, 9, 20, 38, 481344), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:20:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:20:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:20:38,490 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:20:38,568 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:47.679968+00:00 (in 8.003173 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:49 GMT)" (scheduled at 2025-11-04 09:20:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:39,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 39, 677137), True, datetime.datetime(2025, 11, 4, 9, 20, 39, 677137), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:39,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:39,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:39,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:49 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:20:39,686 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:20:39,951 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:40,065 [254] [DEBUG] [app] Starting request: urn:request:52192cc3-901c-44ca-a7a9-332247144e91 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:40,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:40,069 [260] [DEBUG] [app] Starting request: urn:request:945a8913-b585-4d80-8b4f-0ff527b7d5b7 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:40,070 [260] [DEBUG] [app] Ending request: urn:request:945a8913-b585-4d80-8b4f-0ff527b7d5b7 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:945a8913-b585-4d80-8b4f-0ff527b7d5b7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:20:40,070 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:40,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:40,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:40,073 [258] [DEBUG] [app] Starting request: urn:request:6b34aa1f-1755-4245-a3cf-3efdf09809e7 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:40,073 [258] [DEBUG] [app] Ending request: urn:request:6b34aa1f-1755-4245-a3cf-3efdf09809e7 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6b34aa1f-1755-4245-a3cf-3efdf09809e7', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:40,074 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:40,074 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:40,074 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:40,074 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:40,075 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:40,079 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:40,079 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:40,083 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:40,084 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:40,085 [254] [DEBUG] [app] Ending request: urn:request:52192cc3-901c-44ca-a7a9-332247144e91 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:52192cc3-901c-44ca-a7a9-332247144e91', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:40,085 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gunicorn-web stdout | 2025-11-04 09:20:40,085 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:20:40,123 [254] [DEBUG] [app] Starting request: urn:request:db9c4011-b8d4-4806-94f3-28063d8dfaa3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:20:40,127 [250] [DEBUG] [app] Starting request: urn:request:f540e9b2-8f89-40fe-a62d-30b094b29e5f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:40,127 [250] [DEBUG] [app] Ending request: urn:request:f540e9b2-8f89-40fe-a62d-30b094b29e5f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:f540e9b2-8f89-40fe-a62d-30b094b29e5f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:40,128 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:20:40,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:40,129 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:40,131 [257] [DEBUG] [app] Starting request: urn:request:5cecbffd-a220-4ede-8a66-6711debbd77c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:40,131 [257] [DEBUG] [app] Ending request: urn:request:5cecbffd-a220-4ede-8a66-6711debbd77c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:5cecbffd-a220-4ede-8a66-6711debbd77c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:20:40,131 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:20:40,132 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:40,133 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:40,133 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:40,133 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:40,138 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:40,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:40,141 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:40,142 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:40,143 [254] [DEBUG] [app] Ending request: urn:request:db9c4011-b8d4-4806-94f3-28063d8dfaa3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:db9c4011-b8d4-4806-94f3-28063d8dfaa3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:40,143 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:40,144 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) repositorygcworker stdout | 2025-11-04 09:20:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:20:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:06.265965+00:00 (in 26.084834 seconds) repositorygcworker stdout | 2025-11-04 09:20:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:40 GMT)" (scheduled at 2025-11-04 09:20:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:20:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:20:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 40, 181457), True, datetime.datetime(2025, 11, 4, 9, 20, 40, 181457), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:20:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:20:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:20:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:20:40,359 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:20:40,374 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:20:41,179 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:20:41,464 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:20:41,764 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:42,441 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:42,457 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:42,475 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:20:42,864 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:43,266 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:20:43,665 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:20:44,488 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:44,960 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:20:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:20:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:14.980431+00:00 (in 29.999551 seconds) buildlogsarchiver stdout | 2025-11-04 09:20:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:14 GMT)" (scheduled at 2025-11-04 09:20:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:20:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 20, 44, 981139), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:20:44,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:20:44,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:20:44,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:44,989 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:44,992 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:44,995 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:20:45,124 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,479 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,560 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,596 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,629 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,648 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,658 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,670 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,658 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:20:46,685 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:49.676299+00:00 (in 1.995817 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:47 GMT)" (scheduled at 2025-11-04 09:20:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:20:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:57.766253+00:00 (in 9.999594 seconds) notificationworker stdout | 2025-11-04 09:20:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:57 GMT)" (scheduled at 2025-11-04 09:20:47.766253+00:00) notificationworker stdout | 2025-11-04 09:20:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 47, 766979), True, datetime.datetime(2025, 11, 4, 9, 20, 47, 766979), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:47,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:47,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:47,773 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:20:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:20:59.676299+00:00 (in 9.999478 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:49,677 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:59 GMT)" (scheduled at 2025-11-04 09:20:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 49, 677190), True, datetime.datetime(2025, 11, 4, 9, 20, 49, 677190), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:20:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:20:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:20:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:01.707296+00:00 (in 12.000418 seconds) securityworker stdout | 2025-11-04 09:20:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:19 GMT)" (scheduled at 2025-11-04 09:20:49.706478+00:00) securityworker stdout | 2025-11-04 09:20:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:20:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:20:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:20:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:49,714 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:20:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:49,715 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,716 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 19, 29]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-49 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-49 securityworker stdout | 2025-11-04 09:20:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 33, 43]) securityworker stdout | 2025-11-04 09:20:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:20:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:20:49,719 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stdout | 2025-11-04 09:20:49,719 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-33 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 29-33 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 29 securityworker stdout | 2025-11-04 09:20:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 29, 39]) securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 29-39 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 29-39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-29 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-29 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-29 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-43 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 19-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Left range 19-43 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stdout | 2025-11-04 09:20:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 19-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stdout | 2025-11-04 09:20:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 43, 49]) securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-43 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-43 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 19-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 19-43 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-19 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-19 securityworker stdout | 2025-11-04 09:20:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 9 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-19 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-19 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 9 securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 5, 15]) securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 5-15 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 5-15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 5-15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-19 securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-19 securityworker stdout | 2025-11-04 09:20:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,725 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stdout | 2025-11-04 09:20:49,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stdout | 2025-11-04 09:20:49,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 5-15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-5 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 5-15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 15-19 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-19 securityworker stdout | 2025-11-04 09:20:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 15-19 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 15-19 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 15 securityworker stdout | 2025-11-04 09:20:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 15, 25]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 15-25 by worker securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 25-15 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 15-25 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 25 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 25-15 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 8, 18]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 8-18 by worker securityworker stdout | 2025-11-04 09:20:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 8-18 by worker securityworker stdout | 2025-11-04 09:20:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 8-18 securityworker stdout | 2025-11-04 09:20:49,730 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 8-18 securityworker stdout | 2025-11-04 09:20:49,730 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 8-18 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 8-18 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 8-18 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [util.migrate.allocator] Left range 8-18 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-49 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 22, 32]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 8-18 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Right range 8-18 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-8 securityworker stdout | 2025-11-04 09:20:49,732 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-8 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 1, 11]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Merging with block 8-18 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 18 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 8-18 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 18 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Total range: 18-49 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-22 securityworker stdout | 2025-11-04 09:20:49,734 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 18 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 18-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 18-22 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 18 securityworker stdout | 2025-11-04 09:20:49,735 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 18, 28]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 18-28 by worker securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 18-28 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stdout | 2025-11-04 09:20:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 34, 44]) securityworker stdout | 2025-11-04 09:20:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:20:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:20:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 34-44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-34 securityworker stdout | 2025-11-04 09:20:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 32-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 34-44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-34 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 32, 42]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 44 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 34-44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 44-49 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 44-49 securityworker stdout | 2025-11-04 09:20:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stdout | 2025-11-04 09:20:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 44-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 44 securityworker stdout | 2025-11-04 09:20:49,740 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 44, 49]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 44-49 by worker securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 44 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 49-44 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 44-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-44 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:20:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:20:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 31, 41]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 31-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-31 securityworker stdout | 2025-11-04 09:20:49,744 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-31 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 14, 24]) securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 14-24 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:20:49,746 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stdout | 2025-11-04 09:20:49,746 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stdout | 2025-11-04 09:20:49,746 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-31 securityworker stdout | 2025-11-04 09:20:49,746 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Right range 31-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 24-31 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 24 securityworker stdout | 2025-11-04 09:20:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 24, 34]) securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-41 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Left range 14-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-24 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 31-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 14-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 14-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 41-49 securityworker stdout | 2025-11-04 09:20:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 41 securityworker stdout | 2025-11-04 09:20:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 41, 49]) securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 41-49 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 41-49 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 14-41 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-41 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 14 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 14-41 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stdout | 2025-11-04 09:20:49,749 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 4 securityworker stdout | 2025-11-04 09:20:49,750 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 2, 12]) securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 2-12 by worker securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 2-12 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 2-12 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Left range 2-12 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 12-14 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:20:49,751 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:20:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 12, 22]) securityworker stdout | 2025-11-04 09:20:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stdout | 2025-11-04 09:20:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 12-22 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 12-22 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 2-12 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 2-12 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 2 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-2 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-2 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:20:49,753 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 15, 49, 709657), 1, 11]) securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 11 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 11-1 securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:20:49,754 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:20:49,754 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:20:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:20:51,280 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:00.086435+00:00 (in 7.926308 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:52 GMT)" (scheduled at 2025-11-04 09:20:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,166 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,166 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:20:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:20:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:20:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:33.561829+00:00 (in 40.001218 seconds) namespacegcworker stdout | 2025-11-04 09:20:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:53 GMT)" (scheduled at 2025-11-04 09:20:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:20:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:20:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 53, 560986), True, datetime.datetime(2025, 11, 4, 9, 20, 53, 560986), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:20:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:20:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:20:53,569 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:21:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:20:55,065 [257] [DEBUG] [app] Starting request: urn:request:16cb1249-4a06-4ace-b63c-ef214901969a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:55,067 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:55,123 [258] [DEBUG] [app] Starting request: urn:request:ec189728-21ac-40d6-ab3f-318bd496f3d7 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:20:55,124 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:20:55,127 [259] [DEBUG] [app] Starting request: urn:request:5325a82d-a87b-439a-908a-ac91f8fddf93 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:55,127 [259] [DEBUG] [app] Ending request: urn:request:5325a82d-a87b-439a-908a-ac91f8fddf93 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5325a82d-a87b-439a-908a-ac91f8fddf93', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:20:55,127 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:55,128 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:55,129 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:55,131 [258] [DEBUG] [app] Starting request: urn:request:a965e529-f363-4a09-a5f0-d5ed83d4478d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:55,131 [258] [DEBUG] [app] Ending request: urn:request:a965e529-f363-4a09-a5f0-d5ed83d4478d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a965e529-f363-4a09-a5f0-d5ed83d4478d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:20:55,132 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:55,132 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:55,133 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:55,133 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:55,133 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:55,137 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:55,137 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:55,141 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:55,142 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:55,143 [258] [DEBUG] [app] Ending request: urn:request:ec189728-21ac-40d6-ab3f-318bd496f3d7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ec189728-21ac-40d6-ab3f-318bd496f3d7', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:55,143 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:55,144 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) gunicorn-registry stdout | 2025-11-04 09:20:55,218 [250] [DEBUG] [app] Starting request: urn:request:b2349597-e308-43d6-a151-aba6671d03fd (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:20:55,218 [250] [DEBUG] [app] Ending request: urn:request:b2349597-e308-43d6-a151-aba6671d03fd (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b2349597-e308-43d6-a151-aba6671d03fd', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:20:55,219 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:20:55,220 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:55,221 [257] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:20:55,222 [257] [DEBUG] [app] Starting request: urn:request:0bf23cf6-a953-409f-a174-291f85036f2e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:20:55,223 [257] [DEBUG] [app] Ending request: urn:request:0bf23cf6-a953-409f-a174-291f85036f2e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:0bf23cf6-a953-409f-a174-291f85036f2e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:20:55,223 [257] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:20:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:20:55,223 [257] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:20:55,224 [257] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:20:55,224 [257] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:20:55,224 [257] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:20:55,228 [257] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:20:55,228 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:20:55,232 [257] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:20:55,233 [257] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:20:55,234 [257] [DEBUG] [app] Ending request: urn:request:16cb1249-4a06-4ace-b63c-ef214901969a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:16cb1249-4a06-4ace-b63c-ef214901969a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:20:55,234 [257] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:20:55,235 [257] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:20:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:20:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.171 118 0.170) exportactionlogsworker stdout | 2025-11-04 09:20:55,481 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:20:55,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:12.482065+00:00 (in 16.999941 seconds) exportactionlogsworker stdout | 2025-11-04 09:20:55,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:25:55 GMT)" (scheduled at 2025-11-04 09:20:55.481710+00:00) exportactionlogsworker stdout | 2025-11-04 09:20:55,482 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:20:55,489 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), True, datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), 0, 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:20:55,491 [65] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), True, datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), 0, 'exportactionlogs/%', False, datetime.datetime(2025, 11, 4, 9, 20, 55, 482427), 'exportactionlogs/%']) exportactionlogsworker stdout | 2025-11-04 09:20:55,492 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:20:55,492 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:25:55 GMT)" executed successfully gcworker stdout | 2025-11-04 09:20:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:20:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:07.663502+00:00 (in 12.001407 seconds) gcworker stdout | 2025-11-04 09:20:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:25 GMT)" (scheduled at 2025-11-04 09:20:55.661677+00:00) gcworker stdout | 2025-11-04 09:20:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038455662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:20:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:20:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:20:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:20:56,000 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:56,256 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:20:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:20:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) buildlogsarchiver stdout | 2025-11-04 09:20:56,392 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:20:56,604 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:20:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:20:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.000) notificationworker stdout | 2025-11-04 09:20:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:20:57,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:07.766253+00:00 (in 9.999120 seconds) notificationworker stdout | 2025-11-04 09:20:57,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:07 GMT)" (scheduled at 2025-11-04 09:20:57.766253+00:00) notificationworker stdout | 2025-11-04 09:20:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:20:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 57, 767487), True, datetime.datetime(2025, 11, 4, 9, 20, 57, 767487), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:20:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:20:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:20:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:20:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:20:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:09.676299+00:00 (in 9.999506 seconds) proxycacheblobworker stdout | 2025-11-04 09:20:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:09 GMT)" (scheduled at 2025-11-04 09:20:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:20:59,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:20:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 20, 59, 677152), True, datetime.datetime(2025, 11, 4, 9, 20, 59, 677152), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:20:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:20:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:20:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:52.159714+00:00 (in 52.072852 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:00 GMT)" (scheduled at 2025-11-04 09:21:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,095 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:21:00,095 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:21:00,190 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:21:00,288 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:00,576 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:21:01,426 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:21:01,588 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:19.706478+00:00 (in 17.998748 seconds) securityworker stdout | 2025-11-04 09:21:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:31 GMT)" (scheduled at 2025-11-04 09:21:01.707296+00:00) securityworker stdout | 2025-11-04 09:21:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:01,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:01,712 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:01,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:01,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:01,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:01,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:01,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:01,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 1, 711873), 1, 49]) securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:01,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 1, 711873), 1, 49]) securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:01,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:01,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:21:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:21:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:32.081579+00:00 (in 29.999577 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:21:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:32 GMT)" (scheduled at 2025-11-04 09:21:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:21:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:21:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:21:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:21:02,772 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:21:04,385 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:21:04,476 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:21:05,575 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:21:06,067 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:21:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:21:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:40.180734+00:00 (in 33.913767 seconds) repositorygcworker stdout | 2025-11-04 09:21:06,267 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:06 GMT)" (scheduled at 2025-11-04 09:21:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:21:06,267 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:21:06,267 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:21:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:21:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:33.568037+00:00 (in 26.998081 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:21:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:06 GMT)" (scheduled at 2025-11-04 09:21:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:21:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:21:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:21:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:21:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:37.077358+00:00 (in 29.999559 seconds) autopruneworker stdout | 2025-11-04 09:21:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:37 GMT)" (scheduled at 2025-11-04 09:21:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:21:07,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244467082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:21:07,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:21:07,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:21:07,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:21:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:25.661677+00:00 (in 17.997760 seconds) gcworker stdout | 2025-11-04 09:21:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:37 GMT)" (scheduled at 2025-11-04 09:21:07.663502+00:00) gcworker stdout | 2025-11-04 09:21:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:21:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247767670, None, 1, 0]) gcworker stdout | 2025-11-04 09:21:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:07,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:17.766253+00:00 (in 9.999598 seconds) notificationworker stdout | 2025-11-04 09:21:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:17 GMT)" (scheduled at 2025-11-04 09:21:07.766253+00:00) notificationworker stdout | 2025-11-04 09:21:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 7, 766985), True, datetime.datetime(2025, 11, 4, 9, 21, 7, 766985), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:21:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:21:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:07.770322+00:00 (in 59.999565 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:21:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:07 GMT)" (scheduled at 2025-11-04 09:21:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:21:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:21:07,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:21:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:21:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:21:08,579 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:19.676299+00:00 (in 9.999555 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:19 GMT)" (scheduled at 2025-11-04 09:21:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 9, 677017), True, datetime.datetime(2025, 11, 4, 9, 21, 9, 677017), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:19 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:21:09,699 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:09,963 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:10,066 [258] [DEBUG] [app] Starting request: urn:request:c537868c-9d25-46c3-a207-c45b7436d2e7 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:21:10,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:21:10,069 [250] [DEBUG] [app] Starting request: urn:request:df0a2748-5ecf-438f-990c-908f05e90f15 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:10,069 [250] [DEBUG] [app] Ending request: urn:request:df0a2748-5ecf-438f-990c-908f05e90f15 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:df0a2748-5ecf-438f-990c-908f05e90f15', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:10,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:10,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:10,071 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:10,072 [258] [DEBUG] [app] Starting request: urn:request:4c7be79d-d3fc-4d72-ac85-e892a041d4ac (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:10,073 [258] [DEBUG] [app] Ending request: urn:request:4c7be79d-d3fc-4d72-ac85-e892a041d4ac (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4c7be79d-d3fc-4d72-ac85-e892a041d4ac', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:10,073 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:10,073 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:10,074 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:10,074 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:10,074 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:10,077 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:10,078 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:10,081 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:10,082 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:10,083 [258] [DEBUG] [app] Ending request: urn:request:c537868c-9d25-46c3-a207-c45b7436d2e7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c537868c-9d25-46c3-a207-c45b7436d2e7', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:10,083 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:10,083 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gunicorn-web stdout | 2025-11-04 09:21:10,123 [258] [DEBUG] [app] Starting request: urn:request:c34a3548-4677-4916-8bc7-ed81b046b00c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:21:10,129 [250] [DEBUG] [app] Starting request: urn:request:7bfb2d9d-a955-4d88-ab78-ded99aca197a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:10,130 [250] [DEBUG] [app] Ending request: urn:request:7bfb2d9d-a955-4d88-ab78-ded99aca197a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:7bfb2d9d-a955-4d88-ab78-ded99aca197a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:10,130 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:10,130 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:10,131 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:10,132 [254] [DEBUG] [app] Starting request: urn:request:8e7a567d-7497-406e-ace8-3918367f395c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:10,133 [254] [DEBUG] [app] Ending request: urn:request:8e7a567d-7497-406e-ace8-3918367f395c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8e7a567d-7497-406e-ace8-3918367f395c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:10,133 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:10,134 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:10,134 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:10,134 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:10,134 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:10,138 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:10,138 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:10,142 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:10,143 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:10,144 [258] [DEBUG] [app] Ending request: urn:request:c34a3548-4677-4916-8bc7-ed81b046b00c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:c34a3548-4677-4916-8bc7-ed81b046b00c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:10,144 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:10,144 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.022 118 0.022) securityworker stdout | 2025-11-04 09:21:10,373 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:21:10,385 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:21:11,193 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:21:11,476 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:21:11,777 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:12,456 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:21:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:38.480592+00:00 (in 25.998042 seconds) exportactionlogsworker stdout | 2025-11-04 09:21:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:12 GMT)" (scheduled at 2025-11-04 09:21:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:21:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:21:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:12 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:21:12,489 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:12,478 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:21:12,891 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:13,278 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:13,677 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:21:14,501 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:14,976 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:21:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:21:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:44.980431+00:00 (in 29.999568 seconds) buildlogsarchiver stdout | 2025-11-04 09:21:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:44 GMT)" (scheduled at 2025-11-04 09:21:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:21:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 21, 14, 981124), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:21:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:21:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:15,005 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:15,007 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:15,009 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:15,145 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,497 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,574 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,610 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,648 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,661 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,672 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,686 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,690 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:16,703 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:27.766253+00:00 (in 9.999593 seconds) notificationworker stdout | 2025-11-04 09:21:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:27 GMT)" (scheduled at 2025-11-04 09:21:17.766253+00:00) notificationworker stdout | 2025-11-04 09:21:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 17, 766932), True, datetime.datetime(2025, 11, 4, 9, 21, 17, 766932), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:21:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:29.676299+00:00 (in 9.999570 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:29 GMT)" (scheduled at 2025-11-04 09:21:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:19,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:19,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 19, 677465), True, datetime.datetime(2025, 11, 4, 9, 21, 19, 677465), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:21:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:31.707296+00:00 (in 12.000418 seconds) securityworker stdout | 2025-11-04 09:21:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:49 GMT)" (scheduled at 2025-11-04 09:21:19.706478+00:00) securityworker stdout | 2025-11-04 09:21:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:21:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:21:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:21:21,293 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:25,066 [258] [DEBUG] [app] Starting request: urn:request:a71fdd0e-dd38-4573-b4e8-7ed988ec70c1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:21:25,067 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:21:25,069 [250] [DEBUG] [app] Starting request: urn:request:6a6bb5cc-5f1d-4867-8a08-bcd9864c869f (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:25,070 [250] [DEBUG] [app] Ending request: urn:request:6a6bb5cc-5f1d-4867-8a08-bcd9864c869f (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6a6bb5cc-5f1d-4867-8a08-bcd9864c869f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:25,070 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:25,070 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:25,071 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:25,073 [258] [DEBUG] [app] Starting request: urn:request:4348b4c7-bb22-4b9c-b5d5-aa2ac73935c6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:25,073 [258] [DEBUG] [app] Ending request: urn:request:4348b4c7-bb22-4b9c-b5d5-aa2ac73935c6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:4348b4c7-bb22-4b9c-b5d5-aa2ac73935c6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:25,073 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:25,073 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:25,074 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:25,074 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:25,074 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:25,078 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:25,079 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:25,083 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:25,084 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:25,084 [258] [DEBUG] [app] Ending request: urn:request:a71fdd0e-dd38-4573-b4e8-7ed988ec70c1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:a71fdd0e-dd38-4573-b4e8-7ed988ec70c1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:25,085 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:25,085 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) gunicorn-web stdout | 2025-11-04 09:21:25,122 [258] [DEBUG] [app] Starting request: urn:request:3dddde44-ba3d-413c-bf76-7d4d17b48e68 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:21:25,124 [250] [DEBUG] [app] Starting request: urn:request:85f9ffe5-48f3-4b43-bdc6-33fce235c83a (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:25,124 [250] [DEBUG] [app] Ending request: urn:request:85f9ffe5-48f3-4b43-bdc6-33fce235c83a (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:85f9ffe5-48f3-4b43-bdc6-33fce235c83a', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:25,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:25,125 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:25,126 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:25,128 [258] [DEBUG] [app] Starting request: urn:request:cdff2a20-ed89-4e2c-81bd-c71e1204112d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:25,128 [258] [DEBUG] [app] Ending request: urn:request:cdff2a20-ed89-4e2c-81bd-c71e1204112d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:cdff2a20-ed89-4e2c-81bd-c71e1204112d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:25,129 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:25,129 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:25,129 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:25,129 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:25,129 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:25,133 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:25,133 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:25,137 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:25,138 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:25,139 [258] [DEBUG] [app] Ending request: urn:request:3dddde44-ba3d-413c-bf76-7d4d17b48e68 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3dddde44-ba3d-413c-bf76-7d4d17b48e68', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:25,139 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:25,139 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.017) gcworker stdout | 2025-11-04 09:21:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:37.663502+00:00 (in 12.001414 seconds) gcworker stdout | 2025-11-04 09:21:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:55 GMT)" (scheduled at 2025-11-04 09:21:25.661677+00:00) gcworker stdout | 2025-11-04 09:21:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038485662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:21:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:21:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:21:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:26,012 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:21:26,406 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:21:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:27.768179+00:00 (in 0.001482 seconds) notificationworker stdout | 2025-11-04 09:21:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:37 GMT)" (scheduled at 2025-11-04 09:21:27.766253+00:00) notificationworker stdout | 2025-11-04 09:21:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 27, 767051), True, datetime.datetime(2025, 11, 4, 9, 21, 27, 767051), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:37.766253+00:00 (in 9.997784 seconds) notificationworker stdout | 2025-11-04 09:21:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:27 GMT)" (scheduled at 2025-11-04 09:21:27.768179+00:00) notificationworker stdout | 2025-11-04 09:21:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:21:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:39.676299+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:39 GMT)" (scheduled at 2025-11-04 09:21:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 29, 677056), True, datetime.datetime(2025, 11, 4, 9, 21, 29, 677056), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:21:30,203 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:21:30,300 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:21:30,602 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:21:31,452 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:21:31,600 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:21:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:49.706478+00:00 (in 17.998804 seconds) securityworker stdout | 2025-11-04 09:21:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:01 GMT)" (scheduled at 2025-11-04 09:21:31.707296+00:00) securityworker stdout | 2025-11-04 09:21:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:31,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:31,711 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:31,717 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:31,717 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:31,717 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:31,717 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:31,718 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:31,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 31, 711493), 1, 49]) securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:31,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 31, 711493), 1, 49]) securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:31,725 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:21:31,725 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:01 GMT)" executed successfully securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:21:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker pullstatsredisflushworker stdout | 2025-11-04 09:21:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:21:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:02.081579+00:00 (in 29.999577 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:21:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:02 GMT)" (scheduled at 2025-11-04 09:21:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:21:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:21:32,086 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:21:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:21:32,784 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:21:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:21:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:53.560131+00:00 (in 19.997878 seconds) namespacegcworker stdout | 2025-11-04 09:21:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" (scheduled at 2025-11-04 09:21:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:21:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:21:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:21:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:21:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:06.569529+00:00 (in 33.000996 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:21:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" (scheduled at 2025-11-04 09:21:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:21:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:21:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 33, 568989), True, datetime.datetime(2025, 11, 4, 9, 21, 33, 568989), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:21:33,577 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:21:33,577 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:21:33,577 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:21:34,397 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:21:34,503 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:21:35,587 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:21:36,080 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:21:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:21:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:07.077358+00:00 (in 29.999570 seconds) autopruneworker stdout | 2025-11-04 09:21:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:07 GMT)" (scheduled at 2025-11-04 09:21:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:21:37,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244497082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:21:37,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:21:37,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:21:37,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:21:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:55.661677+00:00 (in 17.997764 seconds) gcworker stdout | 2025-11-04 09:21:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:07 GMT)" (scheduled at 2025-11-04 09:21:37.663502+00:00) gcworker stdout | 2025-11-04 09:21:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:21:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247797671, None, 1, 0]) gcworker stdout | 2025-11-04 09:21:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:37,673 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:37,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:47.766253+00:00 (in 9.999121 seconds) notificationworker stdout | 2025-11-04 09:21:37,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:47 GMT)" (scheduled at 2025-11-04 09:21:37.766253+00:00) notificationworker stdout | 2025-11-04 09:21:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 37, 767484), True, datetime.datetime(2025, 11, 4, 9, 21, 37, 767484), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:21:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:21:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:12.482065+00:00 (in 34.000943 seconds) exportactionlogsworker stdout | 2025-11-04 09:21:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:38 GMT)" (scheduled at 2025-11-04 09:21:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:21:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:21:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 38, 481511), True, datetime.datetime(2025, 11, 4, 9, 21, 38, 481511), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:21:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:21:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:21:38,489 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:21:38,591 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:47.679968+00:00 (in 8.003244 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:49 GMT)" (scheduled at 2025-11-04 09:21:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 39, 677382), True, datetime.datetime(2025, 11, 4, 9, 21, 39, 677382), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:49 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:21:39,711 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:21:39,976 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:40,065 [254] [DEBUG] [app] Starting request: urn:request:bff94218-4453-4614-8eee-f4e05e8eac7f (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:21:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:21:40,068 [250] [DEBUG] [app] Starting request: urn:request:ae9ae669-9e1e-40ea-af7e-3ed234cc1eaf (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:40,068 [250] [DEBUG] [app] Ending request: urn:request:ae9ae669-9e1e-40ea-af7e-3ed234cc1eaf (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ae9ae669-9e1e-40ea-af7e-3ed234cc1eaf', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:40,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:40,070 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:40,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:40,072 [261] [DEBUG] [app] Starting request: urn:request:75a523cb-276f-4d7f-880f-06dbadf81469 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:40,073 [261] [DEBUG] [app] Ending request: urn:request:75a523cb-276f-4d7f-880f-06dbadf81469 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:75a523cb-276f-4d7f-880f-06dbadf81469', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:40,073 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:40,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:21:40,073 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:40,073 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:40,073 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:40,077 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:40,077 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:40,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:40,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:40,083 [254] [DEBUG] [app] Ending request: urn:request:bff94218-4453-4614-8eee-f4e05e8eac7f (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:bff94218-4453-4614-8eee-f4e05e8eac7f', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:40,083 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.019) gunicorn-web stdout | 2025-11-04 09:21:40,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" gunicorn-web stdout | 2025-11-04 09:21:40,123 [258] [DEBUG] [app] Starting request: urn:request:89786325-2b9a-432b-b2ef-ba961e886ca3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:21:40,125 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:21:40,128 [250] [DEBUG] [app] Starting request: urn:request:c2220f57-087b-4b8d-80c3-0a497114a43c (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:40,129 [250] [DEBUG] [app] Ending request: urn:request:c2220f57-087b-4b8d-80c3-0a497114a43c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:c2220f57-087b-4b8d-80c3-0a497114a43c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:40,129 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:40,129 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:21:40,130 [258] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:40,132 [254] [DEBUG] [app] Starting request: urn:request:47e1714d-bba2-44ec-b3b8-858d1174a106 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:40,132 [254] [DEBUG] [app] Ending request: urn:request:47e1714d-bba2-44ec-b3b8-858d1174a106 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:47e1714d-bba2-44ec-b3b8-858d1174a106', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:40,132 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:40,132 [258] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:40,133 [258] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:40,133 [258] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:40,133 [258] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:40,136 [258] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:40,136 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:40,140 [258] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:40,141 [258] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:40,142 [258] [DEBUG] [app] Ending request: urn:request:89786325-2b9a-432b-b2ef-ba961e886ca3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:89786325-2b9a-432b-b2ef-ba961e886ca3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:40,142 [258] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:40,142 [258] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.021 118 0.021) repositorygcworker stdout | 2025-11-04 09:21:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:21:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:06.265965+00:00 (in 26.084788 seconds) repositorygcworker stdout | 2025-11-04 09:21:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:40 GMT)" (scheduled at 2025-11-04 09:21:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:21:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:21:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 40, 181552), True, datetime.datetime(2025, 11, 4, 9, 21, 40, 181552), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:21:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:21:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:21:40,189 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:21:40,386 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:21:40,398 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:21:41,205 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:21:41,489 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:21:41,789 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:42,490 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:42,507 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:42,513 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:21:42,903 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:43,292 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:21:43,690 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:21:44,514 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:21:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:21:44,981 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:14.980431+00:00 (in 29.999212 seconds) buildlogsarchiver stdout | 2025-11-04 09:21:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:14 GMT)" (scheduled at 2025-11-04 09:21:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:21:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 21, 44, 981533), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:21:44,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:21:44,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:21:44,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:44,991 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:45,023 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:45,024 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:45,029 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:21:45,160 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,514 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,588 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,628 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,664 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,675 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,689 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,702 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,708 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:21:46,720 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:49.676299+00:00 (in 1.995911 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:47 GMT)" (scheduled at 2025-11-04 09:21:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:47,680 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:21:47,680 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:57.766253+00:00 (in 9.999590 seconds) notificationworker stdout | 2025-11-04 09:21:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:57 GMT)" (scheduled at 2025-11-04 09:21:47.766253+00:00) notificationworker stdout | 2025-11-04 09:21:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 47, 766991), True, datetime.datetime(2025, 11, 4, 9, 21, 47, 766991), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:47,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:47,773 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:47,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:21:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:21:59.676299+00:00 (in 9.999554 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:59 GMT)" (scheduled at 2025-11-04 09:21:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 49, 677101), True, datetime.datetime(2025, 11, 4, 9, 21, 49, 677101), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:21:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:21:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:21:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:01.707296+00:00 (in 12.000413 seconds) securityworker stdout | 2025-11-04 09:21:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:19 GMT)" (scheduled at 2025-11-04 09:21:49.706478+00:00) securityworker stdout | 2025-11-04 09:21:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:21:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:21:49,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:21:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:21:49,714 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:21:49,715 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,715 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,715 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:49,715 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,716 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 22, 32]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-49 securityworker stdout | 2025-11-04 09:21:49,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 33, 43]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 33-43 by worker securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 33-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 33-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:21:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:21:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 11, 21]) securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 3 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 4 total holes securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-22 securityworker stdout | 2025-11-04 09:21:49,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-22 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:21:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 21, 31]) securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-21 securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-21 securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-21 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-32 securityworker stdout | 2025-11-04 09:21:49,725 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-32 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-32 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Left range 11-32 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stdout | 2025-11-04 09:21:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 33-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-33 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:21:49,727 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-32 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-32 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 33-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,728 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,729 [92] [DEBUG] [util.migrate.allocator] Right range 11-43 securityworker stdout | 2025-11-04 09:21:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stdout | 2025-11-04 09:21:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-43 securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 43 securityworker stdout | 2025-11-04 09:21:49,730 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 43-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 43-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stdout | 2025-11-04 09:21:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 43-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 43 securityworker stdout | 2025-11-04 09:21:49,732 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 43, 49]) securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 43 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 49-43 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 43-49 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 43-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-43 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,734 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 24, 34]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:21:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 24-34 by worker securityworker stdout | 2025-11-04 09:21:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stdout | 2025-11-04 09:21:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 24-34 securityworker stdout | 2025-11-04 09:21:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,736 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,736 [92] [DEBUG] [util.migrate.allocator] Right range 24-34 securityworker stdout | 2025-11-04 09:21:49,736 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-24 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-24 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:21:49,736 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 14 securityworker stdout | 2025-11-04 09:21:49,737 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 13, 23]) securityworker stdout | 2025-11-04 09:21:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stdout | 2025-11-04 09:21:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stdout | 2025-11-04 09:21:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Left range 13-23 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Right range 24-34 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-24 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 13-23 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 13-23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 13-23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 23-24 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 23 securityworker stdout | 2025-11-04 09:21:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 23, 33]) securityworker stdout | 2025-11-04 09:21:49,739 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stdout | 2025-11-04 09:21:49,739 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stdout | 2025-11-04 09:21:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 13-23 securityworker stdout | 2025-11-04 09:21:49,739 [92] [DEBUG] [util.migrate.allocator] Already merged with block 13-23 securityworker stdout | 2025-11-04 09:21:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 23-33 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 23-33 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 13-23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 13-23 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 24-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-34 securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Right range 13-34 securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-13 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 13-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 13-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-13 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 3 securityworker stdout | 2025-11-04 09:21:49,740 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 3 securityworker stdout | 2025-11-04 09:21:49,741 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 3, 13]) securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-34 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-34 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 3-13 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 3-13 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 13-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 3-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Right range 3-34 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stdout | 2025-11-04 09:21:49,742 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 3-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-3 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:49,743 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 1, 11]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-34 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 34 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 34-49 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 3-34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 34 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 34-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 34-49 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 34-49 securityworker stdout | 2025-11-04 09:21:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 37, 47]) securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Total range: 34-49 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 34-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 34-37 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 34 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 34-37 securityworker stdout | 2025-11-04 09:21:49,747 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 34 securityworker stdout | 2025-11-04 09:21:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 34, 44]) securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 34-44 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 34-44 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,749 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stdout | 2025-11-04 09:21:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:21:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:21:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 47, 49]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:21:49,751 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:21:49,751 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:21:49,751 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,752 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 7, 17]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 7-17 by worker securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 7-17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 7-17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Right range 7-17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-7 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:49,754 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:21:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 1, 11]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 7-17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-49 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-49 securityworker stdout | 2025-11-04 09:21:49,756 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 35, 45]) securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 35-45 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 35-45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 35-45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stdout | 2025-11-04 09:21:49,757 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stdout | 2025-11-04 09:21:49,758 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 35-45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 45-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 45 securityworker stdout | 2025-11-04 09:21:49,758 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 45, 49]) securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 45-49 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 45-49 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 35-45 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Total range: 17-35 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 35-45 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:21:49,759 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 25 securityworker stdout | 2025-11-04 09:21:49,760 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 22, 32]) securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Total range: 17-35 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-35 securityworker stdout | 2025-11-04 09:21:49,761 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-35 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 32, 42]) securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 22 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 22 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 17-22 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Total range: 17-22 securityworker stdout | 2025-11-04 09:21:49,762 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:21:49,763 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-22 securityworker stdout | 2025-11-04 09:21:49,763 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 17-22 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 17 securityworker stdout | 2025-11-04 09:21:49,763 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 16, 49, 709775), 17, 27]) securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 17-27 by worker securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 17-27 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 27 securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:21:49,764 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:21:49,764 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:19 GMT)" executed successfully securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] Total range: 27-17 securityworker stderr | 2025-11-04 09:21:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker storagereplication stdout | 2025-11-04 09:21:51,306 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,160 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:00.086435+00:00 (in 7.925969 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:52 GMT)" (scheduled at 2025-11-04 09:21:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:21:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:21:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:21:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:32.561493+00:00 (in 39.000910 seconds) namespacegcworker stdout | 2025-11-04 09:21:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:53 GMT)" (scheduled at 2025-11-04 09:21:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:21:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:21:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 53, 560858), True, datetime.datetime(2025, 11, 4, 9, 21, 53, 560858), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:21:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:21:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:21:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:22:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:21:55,065 [261] [DEBUG] [app] Starting request: urn:request:8dcd5ed1-9065-4178-a603-e30f618a2ee3 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:21:55,067 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:21:55,069 [260] [DEBUG] [app] Starting request: urn:request:b921e995-ba48-426a-b0c5-45aae82fcdc0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:55,069 [260] [DEBUG] [app] Ending request: urn:request:b921e995-ba48-426a-b0c5-45aae82fcdc0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b921e995-ba48-426a-b0c5-45aae82fcdc0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:55,069 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:21:55,069 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:55,070 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:55,072 [254] [DEBUG] [app] Starting request: urn:request:8ca7b3be-dea3-4716-922b-a1e89aad0e46 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:55,073 [254] [DEBUG] [app] Ending request: urn:request:8ca7b3be-dea3-4716-922b-a1e89aad0e46 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:8ca7b3be-dea3-4716-922b-a1e89aad0e46', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:55,073 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:21:55,073 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:55,074 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:55,074 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:55,074 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:55,078 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:55,078 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:55,082 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:55,083 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:55,084 [261] [DEBUG] [app] Ending request: urn:request:8dcd5ed1-9065-4178-a603-e30f618a2ee3 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:8dcd5ed1-9065-4178-a603-e30f618a2ee3', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:55,084 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:55,084 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:21:55,122 [261] [DEBUG] [app] Starting request: urn:request:2a1db2e2-27f1-4793-af51-f377824f16be (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:21:55,124 [260] [DEBUG] [app] Starting request: urn:request:71b10e82-8931-46c7-b77c-216a2cfe34bc (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:21:55,124 [260] [DEBUG] [app] Ending request: urn:request:71b10e82-8931-46c7-b77c-216a2cfe34bc (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:71b10e82-8931-46c7-b77c-216a2cfe34bc', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:21:55,124 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:21:55,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:55,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:21:55,127 [258] [DEBUG] [app] Starting request: urn:request:92968203-14fc-4569-ba42-46f371d9391b (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:21:55,128 [258] [DEBUG] [app] Ending request: urn:request:92968203-14fc-4569-ba42-46f371d9391b (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:92968203-14fc-4569-ba42-46f371d9391b', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:21:55,128 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:21:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.003 159 0.002) gunicorn-web stdout | 2025-11-04 09:21:55,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:21:55,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:21:55,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:21:55,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:21:55,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:21:55,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:21:55,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:21:55,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:21:55,139 [261] [DEBUG] [app] Ending request: urn:request:2a1db2e2-27f1-4793-af51-f377824f16be (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2a1db2e2-27f1-4793-af51-f377824f16be', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:21:55,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:21:55,139 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:21:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:21:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) gcworker stdout | 2025-11-04 09:21:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:21:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:07.663502+00:00 (in 12.001418 seconds) gcworker stdout | 2025-11-04 09:21:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:25 GMT)" (scheduled at 2025-11-04 09:21:55.661677+00:00) gcworker stdout | 2025-11-04 09:21:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038515662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:21:55,672 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:21:55,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:21:55,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:21:56,025 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:56,203 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:21:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.131.0.44 (-) - - [04/Nov/2025:09:21:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) buildlogsarchiver stdout | 2025-11-04 09:21:56,418 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:21:56,598 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:21:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:21:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) notificationworker stdout | 2025-11-04 09:21:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:21:57,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:07.766253+00:00 (in 9.999103 seconds) notificationworker stdout | 2025-11-04 09:21:57,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:07 GMT)" (scheduled at 2025-11-04 09:21:57.766253+00:00) notificationworker stdout | 2025-11-04 09:21:57,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:21:57,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 57, 767560), True, datetime.datetime(2025, 11, 4, 9, 21, 57, 767560), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:21:57,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:21:57,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:21:57,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:21:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:21:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:09.676299+00:00 (in 9.999575 seconds) proxycacheblobworker stdout | 2025-11-04 09:21:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:09 GMT)" (scheduled at 2025-11-04 09:21:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:21:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:21:59,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 21, 59, 677044), True, datetime.datetime(2025, 11, 4, 9, 21, 59, 677044), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:21:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:21:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:21:59,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:52.159714+00:00 (in 52.072846 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:00 GMT)" (scheduled at 2025-11-04 09:22:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,093 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,093 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:22:00,093 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:22:00,215 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:22:00,313 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:00,614 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:22:01,465 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:22:01,613 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:19.706478+00:00 (in 17.998778 seconds) securityworker stdout | 2025-11-04 09:22:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:31 GMT)" (scheduled at 2025-11-04 09:22:01.707296+00:00) securityworker stdout | 2025-11-04 09:22:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:01,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:01,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:01,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:01,721 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 1, 713589), 1, 49]) securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 1, 713589), 1, 49]) securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:01,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:01,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:22:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:22:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:32.081579+00:00 (in 29.999574 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:22:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:32 GMT)" (scheduled at 2025-11-04 09:22:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:22:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:22:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:22:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:22:02,797 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:22:04,410 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:22:04,515 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:22:05,599 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:22:06,094 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:22:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:22:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:40.180734+00:00 (in 33.914103 seconds) repositorygcworker stdout | 2025-11-04 09:22:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:06 GMT)" (scheduled at 2025-11-04 09:22:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:22:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:22:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:06,570 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:28.569177+00:00 (in 21.999071 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:06 GMT)" (scheduled at 2025-11-04 09:22:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:22:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:22:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:22:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:37.077358+00:00 (in 29.999567 seconds) autopruneworker stdout | 2025-11-04 09:22:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:37 GMT)" (scheduled at 2025-11-04 09:22:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:22:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244527082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:22:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:22:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:22:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:22:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:25.661677+00:00 (in 17.997738 seconds) gcworker stdout | 2025-11-04 09:22:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:37 GMT)" (scheduled at 2025-11-04 09:22:07.663502+00:00) gcworker stdout | 2025-11-04 09:22:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:22:07,670 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247827670, None, 1, 0]) gcworker stdout | 2025-11-04 09:22:07,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:07,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:17.766253+00:00 (in 9.999359 seconds) notificationworker stdout | 2025-11-04 09:22:07,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:17 GMT)" (scheduled at 2025-11-04 09:22:07.766253+00:00) notificationworker stdout | 2025-11-04 09:22:07,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 7, 767172), True, datetime.datetime(2025, 11, 4, 9, 22, 7, 767172), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:22:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:22:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:07.770322+00:00 (in 59.999532 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:22:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:07 GMT)" (scheduled at 2025-11-04 09:22:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:22:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:22:07,773 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:07,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:07,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:22:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:22:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:22:08,604 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:19.676299+00:00 (in 9.999599 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:19 GMT)" (scheduled at 2025-11-04 09:22:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:09,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:09,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 9, 677036), True, datetime.datetime(2025, 11, 4, 9, 22, 9, 677036), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:09,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:09,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:09,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:19 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:22:09,724 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:09,989 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:10,065 [261] [DEBUG] [app] Starting request: urn:request:9acbe984-6123-483d-a619-22c602a1be71 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:22:10,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:22:10,068 [256] [DEBUG] [app] Starting request: urn:request:5ee1abe5-8f52-4584-b8f4-709eec9b2df2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:10,069 [256] [DEBUG] [app] Ending request: urn:request:5ee1abe5-8f52-4584-b8f4-709eec9b2df2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5ee1abe5-8f52-4584-b8f4-709eec9b2df2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:10,069 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:10,069 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:22:10,076 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:10,077 [254] [DEBUG] [app] Starting request: urn:request:2ee5a332-dc03-4182-8556-d399f6ebc771 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:10,077 [254] [DEBUG] [app] Ending request: urn:request:2ee5a332-dc03-4182-8556-d399f6ebc771 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:2ee5a332-dc03-4182-8556-d399f6ebc771', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:10,078 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:10,078 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:10,078 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:10,078 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:10,079 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:10,082 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:10,083 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:10,086 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:10,087 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:10,088 [261] [DEBUG] [app] Ending request: urn:request:9acbe984-6123-483d-a619-22c602a1be71 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:9acbe984-6123-483d-a619-22c602a1be71', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:10,088 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:10,089 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.024 118 0.024) gunicorn-web stdout | 2025-11-04 09:22:10,122 [261] [DEBUG] [app] Starting request: urn:request:1e86010e-187b-45f3-9b32-b8bac2a37bb7 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:22:10,124 [250] [DEBUG] [app] Starting request: urn:request:e34fc239-4420-4070-818b-0a24ec73356c (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:10,124 [250] [DEBUG] [app] Ending request: urn:request:e34fc239-4420-4070-818b-0a24ec73356c (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e34fc239-4420-4070-818b-0a24ec73356c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:22:10,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:10,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:10,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:10,128 [254] [DEBUG] [app] Starting request: urn:request:f16f4030-9b2b-46cf-bd89-f850e1e1c705 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:10,128 [254] [DEBUG] [app] Ending request: urn:request:f16f4030-9b2b-46cf-bd89-f850e1e1c705 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f16f4030-9b2b-46cf-bd89-f850e1e1c705', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:10,128 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:10,128 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:10,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:10,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:10,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:10,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:10,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:10,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:10,138 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:10,139 [261] [DEBUG] [app] Ending request: urn:request:1e86010e-187b-45f3-9b32-b8bac2a37bb7 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:1e86010e-187b-45f3-9b32-b8bac2a37bb7', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:10,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:10,139 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) securityworker stdout | 2025-11-04 09:22:10,398 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:22:10,412 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:22:11,217 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:22:11,501 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:22:11,801 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:22:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:38.480592+00:00 (in 25.997999 seconds) exportactionlogsworker stdout | 2025-11-04 09:22:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:12 GMT)" (scheduled at 2025-11-04 09:22:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:22:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:22:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:12 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:22:12,505 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:12,521 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:12,527 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:22:12,915 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:13,319 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:13,702 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:22:14,527 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:22:14,981 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:22:14,981 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:44.980431+00:00 (in 29.999198 seconds) buildlogsarchiver stdout | 2025-11-04 09:22:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:44 GMT)" (scheduled at 2025-11-04 09:22:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:22:14,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 22, 14, 981523), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:14,989 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:22:14,989 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:22:14,989 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:15,007 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:15,036 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:15,042 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:15,042 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:15,174 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,530 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,607 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,646 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,682 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,690 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,707 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,716 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,724 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:16,737 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:19.767846+00:00 (in 2.001166 seconds) notificationworker stdout | 2025-11-04 09:22:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:27 GMT)" (scheduled at 2025-11-04 09:22:17.766253+00:00) notificationworker stdout | 2025-11-04 09:22:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:17,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 17, 767072), True, datetime.datetime(2025, 11, 4, 9, 22, 17, 767072), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:17,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:17,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:17,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:22:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:29.676299+00:00 (in 9.999566 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:29 GMT)" (scheduled at 2025-11-04 09:22:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 19, 677061), True, datetime.datetime(2025, 11, 4, 9, 22, 19, 677061), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:22:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:31.707296+00:00 (in 12.000418 seconds) securityworker stdout | 2025-11-04 09:22:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:49 GMT)" (scheduled at 2025-11-04 09:22:19.706478+00:00) securityworker stdout | 2025-11-04 09:22:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:19,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:19,715 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:22:19,715 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:49 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:19,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:19,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:27.766253+00:00 (in 7.998011 seconds) notificationworker stdout | 2025-11-04 09:22:19,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:19 GMT)" (scheduled at 2025-11-04 09:22:19.767846+00:00) notificationworker stdout | 2025-11-04 09:22:19,769 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), 'notification/%']) notificationworker stdout | 2025-11-04 09:22:19,775 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), True, datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), 0, 'notification/%']) notificationworker stdout | 2025-11-04 09:22:19,776 [77] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), True, datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), 0, 'notification/%', False, datetime.datetime(2025, 11, 4, 9, 22, 19, 768586), 'notification/%']) notificationworker stdout | 2025-11-04 09:22:19,778 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:19,778 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:22:21,333 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:25,065 [261] [DEBUG] [app] Starting request: urn:request:e7134a7d-5523-41a3-83f5-c931ff6ecbc5 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:22:25,066 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:22:25,067 [256] [DEBUG] [app] Starting request: urn:request:6b0e3e8a-a6e5-46a9-8f3b-328f40c35086 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:25,068 [256] [DEBUG] [app] Ending request: urn:request:6b0e3e8a-a6e5-46a9-8f3b-328f40c35086 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6b0e3e8a-a6e5-46a9-8f3b-328f40c35086', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:25,068 [256] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:22:25,068 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:25,069 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:25,071 [254] [DEBUG] [app] Starting request: urn:request:e94b334e-8e70-42e3-9b2b-97b820ae6d0c (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:25,071 [254] [DEBUG] [app] Ending request: urn:request:e94b334e-8e70-42e3-9b2b-97b820ae6d0c (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:e94b334e-8e70-42e3-9b2b-97b820ae6d0c', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:25,071 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:25,071 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:25,072 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:25,072 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:25,072 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:25,076 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:25,076 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:25,080 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:25,081 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:25,082 [261] [DEBUG] [app] Ending request: urn:request:e7134a7d-5523-41a3-83f5-c931ff6ecbc5 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:e7134a7d-5523-41a3-83f5-c931ff6ecbc5', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:25,082 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:25,082 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.018 118 0.018) gunicorn-web stdout | 2025-11-04 09:22:25,123 [254] [DEBUG] [app] Starting request: urn:request:7efa1d3c-8e6b-4f2e-8e7b-244c0b9d321c (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:22:25,130 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:22:25,133 [260] [DEBUG] [app] Starting request: urn:request:bb090a39-4ee5-4c61-a501-26a957a31924 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:25,133 [260] [DEBUG] [app] Ending request: urn:request:bb090a39-4ee5-4c61-a501-26a957a31924 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:bb090a39-4ee5-4c61-a501-26a957a31924', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-registry stdout | 2025-11-04 09:22:25,133 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:25,134 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:25,134 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:25,136 [254] [DEBUG] [app] Starting request: urn:request:a8812e5e-cdbd-4079-829b-f556eb926cb1 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:25,136 [254] [DEBUG] [app] Ending request: urn:request:a8812e5e-cdbd-4079-829b-f556eb926cb1 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:a8812e5e-cdbd-4079-829b-f556eb926cb1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:25,136 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:25,137 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:25,137 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:25,137 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:25,137 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:25,141 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:25,141 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:25,146 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:25,147 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:25,147 [254] [DEBUG] [app] Ending request: urn:request:7efa1d3c-8e6b-4f2e-8e7b-244c0b9d321c (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:7efa1d3c-8e6b-4f2e-8e7b-244c0b9d321c', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:25,147 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:25,148 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.026 118 0.025) gcworker stdout | 2025-11-04 09:22:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:37.663502+00:00 (in 12.001373 seconds) gcworker stdout | 2025-11-04 09:22:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:55 GMT)" (scheduled at 2025-11-04 09:22:25.661677+00:00) gcworker stdout | 2025-11-04 09:22:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038545662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:22:25,670 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:22:25,670 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:25,670 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:22:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:26,037 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:22:26,431 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:22:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:27,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:27.768179+00:00 (in 0.001491 seconds) notificationworker stdout | 2025-11-04 09:22:27,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:37 GMT)" (scheduled at 2025-11-04 09:22:27.766253+00:00) notificationworker stdout | 2025-11-04 09:22:27,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:27,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 27, 767024), True, datetime.datetime(2025, 11, 4, 9, 22, 27, 767024), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:37.766253+00:00 (in 9.997748 seconds) notificationworker stdout | 2025-11-04 09:22:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:27 GMT)" (scheduled at 2025-11-04 09:22:27.768179+00:00) notificationworker stdout | 2025-11-04 09:22:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:22:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:27,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:27,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:27,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:37 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:28,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:28,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:33.568037+00:00 (in 4.998379 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:28,569 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:28 GMT)" (scheduled at 2025-11-04 09:22:28.569177+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:28,570 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:28,577 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), True, datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), 0, 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:28,586 [91] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), True, datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), 0, 'secscanv4/%', False, datetime.datetime(2025, 11, 4, 9, 22, 28, 569988), 'secscanv4/%']) securityscanningnotificationworker stdout | 2025-11-04 09:22:28,587 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:22:28,588 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:28 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:39.676299+00:00 (in 9.999581 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:39 GMT)" (scheduled at 2025-11-04 09:22:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 29, 677043), True, datetime.datetime(2025, 11, 4, 9, 22, 29, 677043), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:29,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:29,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:29,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:22:30,228 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:22:30,325 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:22:30,627 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:22:31,483 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:22:31,626 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:22:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:49.706478+00:00 (in 17.998745 seconds) securityworker stdout | 2025-11-04 09:22:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:01 GMT)" (scheduled at 2025-11-04 09:22:31.707296+00:00) securityworker stdout | 2025-11-04 09:22:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:31,710 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:31,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stdout | 2025-11-04 09:22:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:31,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 31, 712875), 1, 49]) securityworker stdout | 2025-11-04 09:22:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:31,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 31, 712875), 1, 49]) securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:22:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:22:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:22:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:02.081579+00:00 (in 29.999583 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:22:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:02 GMT)" (scheduled at 2025-11-04 09:22:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:22:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:22:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:22:32,086 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:02 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:22:32,561 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:22:32,561 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:33.561829+00:00 (in 0.999897 seconds) namespacegcworker stdout | 2025-11-04 09:22:32,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:32 GMT)" (scheduled at 2025-11-04 09:22:32.561493+00:00) namespacegcworker stdout | 2025-11-04 09:22:32,562 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [False, datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:22:32,569 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), True, datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), 0, 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:22:32,571 [75] [DEBUG] [peewee] ('SELECT COUNT(1) FROM (SELECT DISTINCT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) AND NOT ("t1"."queue_name" IN (SELECT "t1"."queue_name" FROM "queueitem" AS "t1" WHERE ((("t1"."available" = %s) AND ("t1"."processing_expires" > %s)) AND ("t1"."queue_name" ILIKE %s)))))) AS "_wrapped"', [datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), True, datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), 0, 'namespacegc/%', False, datetime.datetime(2025, 11, 4, 9, 22, 32, 562172), 'namespacegc/%']) namespacegcworker stdout | 2025-11-04 09:22:32,572 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:22:32,572 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.update_queue_metrics (trigger: interval[0:05:00], next run at: 2025-11-04 09:27:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:22:32,810 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:22:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:22:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:53.560131+00:00 (in 19.997838 seconds) namespacegcworker stdout | 2025-11-04 09:22:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" (scheduled at 2025-11-04 09:22:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:22:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:22:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:22:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:22:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:06.569529+00:00 (in 33.001033 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:22:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" (scheduled at 2025-11-04 09:22:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:22:33,569 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:22:33,575 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 33, 569246), True, datetime.datetime(2025, 11, 4, 9, 22, 33, 569246), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:22:33,583 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:22:33,583 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:22:33,583 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:22:34,437 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:22:34,528 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:22:35,612 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:22:36,121 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:22:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:22:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:07.077358+00:00 (in 29.999567 seconds) autopruneworker stdout | 2025-11-04 09:22:37,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:07 GMT)" (scheduled at 2025-11-04 09:22:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:22:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244557082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:22:37,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:22:37,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:22:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:22:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:55.661677+00:00 (in 17.997769 seconds) gcworker stdout | 2025-11-04 09:22:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:07 GMT)" (scheduled at 2025-11-04 09:22:37.663502+00:00) gcworker stdout | 2025-11-04 09:22:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:22:37,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247857670, None, 1, 0]) gcworker stdout | 2025-11-04 09:22:37,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:37,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:47.766253+00:00 (in 9.999578 seconds) notificationworker stdout | 2025-11-04 09:22:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:47 GMT)" (scheduled at 2025-11-04 09:22:37.766253+00:00) notificationworker stdout | 2025-11-04 09:22:37,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:37,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 37, 767057), True, datetime.datetime(2025, 11, 4, 9, 22, 37, 767057), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:37,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:37,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:37,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:22:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:22:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:12.482065+00:00 (in 34.001020 seconds) exportactionlogsworker stdout | 2025-11-04 09:22:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:38 GMT)" (scheduled at 2025-11-04 09:22:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:22:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:22:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 38, 481305), True, datetime.datetime(2025, 11, 4, 9, 22, 38, 481305), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:22:38,488 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:22:38,488 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:22:38,488 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:22:38,617 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:47.679968+00:00 (in 8.003251 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:49 GMT)" (scheduled at 2025-11-04 09:22:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:39,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 39, 677034), True, datetime.datetime(2025, 11, 4, 9, 22, 39, 677034), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:39,683 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:39,683 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:39,683 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:49 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:22:39,736 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:22:40,001 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:40,065 [254] [DEBUG] [app] Starting request: urn:request:64c7093a-840c-4113-ad23-39a22fdb254d (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:22:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:22:40,068 [250] [DEBUG] [app] Starting request: urn:request:03eab6e5-64b4-42f3-967a-6ea1ecd3bac8 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:40,068 [250] [DEBUG] [app] Ending request: urn:request:03eab6e5-64b4-42f3-967a-6ea1ecd3bac8 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:03eab6e5-64b4-42f3-967a-6ea1ecd3bac8', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:40,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:22:40,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:40,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:40,071 [254] [DEBUG] [app] Starting request: urn:request:3ae9a50a-6cf0-47a5-9aae-924d06064b97 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:40,071 [254] [DEBUG] [app] Ending request: urn:request:3ae9a50a-6cf0-47a5-9aae-924d06064b97 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3ae9a50a-6cf0-47a5-9aae-924d06064b97', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:40,072 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:40,072 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:40,072 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:40,072 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:40,072 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:40,076 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:40,076 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:40,081 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:40,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:40,082 [254] [DEBUG] [app] Ending request: urn:request:64c7093a-840c-4113-ad23-39a22fdb254d (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:64c7093a-840c-4113-ad23-39a22fdb254d', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:40,082 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:40,083 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.018) gunicorn-web stdout | 2025-11-04 09:22:40,123 [254] [DEBUG] [app] Starting request: urn:request:774ae98f-6ab0-4c1e-88ae-cf0f57322add (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:22:40,125 [250] [DEBUG] [app] Starting request: urn:request:5529e5d2-cae4-475f-922e-1065902a5c66 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:40,126 [250] [DEBUG] [app] Ending request: urn:request:5529e5d2-cae4-475f-922e-1065902a5c66 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:5529e5d2-cae4-475f-922e-1065902a5c66', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:22:40,126 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:40,126 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:40,127 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:40,128 [261] [DEBUG] [app] Starting request: urn:request:9c7921dd-82b6-4d3d-86dc-11ad822e7df6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:40,129 [261] [DEBUG] [app] Ending request: urn:request:9c7921dd-82b6-4d3d-86dc-11ad822e7df6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:9c7921dd-82b6-4d3d-86dc-11ad822e7df6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:40,129 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:40,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:22:40,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:40,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:40,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:40,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:40,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:40,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:40,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:40,138 [254] [DEBUG] [app] Ending request: urn:request:774ae98f-6ab0-4c1e-88ae-cf0f57322add (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:774ae98f-6ab0-4c1e-88ae-cf0f57322add', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:40,138 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.015 118 0.016) gunicorn-web stdout | 2025-11-04 09:22:40,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:22:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:22:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:06.265965+00:00 (in 26.084825 seconds) repositorygcworker stdout | 2025-11-04 09:22:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:40 GMT)" (scheduled at 2025-11-04 09:22:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:22:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:22:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 40, 181489), True, datetime.datetime(2025, 11, 4, 9, 22, 40, 181489), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:22:40,187 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:22:40,187 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:22:40,187 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:22:40,411 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:22:40,439 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:22:41,230 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:22:41,514 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:22:41,814 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:42,519 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:42,535 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:42,542 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:22:42,928 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:43,331 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:22:43,714 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:22:44,540 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:22:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:22:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:14.980431+00:00 (in 29.999574 seconds) buildlogsarchiver stdout | 2025-11-04 09:22:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:14 GMT)" (scheduled at 2025-11-04 09:22:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:22:44,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 22, 44, 981129), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:22:44,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:22:44,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:22:44,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:45,022 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:45,051 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:45,055 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:45,056 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:22:45,189 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,546 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,625 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,662 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,699 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,707 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,724 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,731 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,740 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:22:46,753 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:47,680 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:47,680 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:49.676299+00:00 (in 1.995911 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:47,680 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:47 GMT)" (scheduled at 2025-11-04 09:22:47.679968+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:47,681 [78] [DEBUG] [workers.queueworker] Running watchdog. proxycacheblobworker stdout | 2025-11-04 09:22:47,681 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:47 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:47,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:47,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:57.766253+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 09:22:47,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:57 GMT)" (scheduled at 2025-11-04 09:22:47.766253+00:00) notificationworker stdout | 2025-11-04 09:22:47,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:47,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 47, 767055), True, datetime.datetime(2025, 11, 4, 9, 22, 47, 767055), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:47,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:47,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:47,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:57 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:22:49,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:49,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:22:59.676299+00:00 (in 9.999542 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:49,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:59 GMT)" (scheduled at 2025-11-04 09:22:49.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:49,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:49,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 49, 677081), True, datetime.datetime(2025, 11, 4, 9, 22, 49, 677081), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:49,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:49,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:49,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:22:59 GMT)" executed successfully securityworker stdout | 2025-11-04 09:22:49,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:22:49,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:01.707296+00:00 (in 12.000393 seconds) securityworker stdout | 2025-11-04 09:22:49,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:19 GMT)" (scheduled at 2025-11-04 09:22:49.706478+00:00) securityworker stdout | 2025-11-04 09:22:49,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:22:49,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:22:49,709 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:22:49,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:49,716 [92] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:22:49,716 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,716 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:49,716 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:49,716 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:22:49,717 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 37, 47]) securityworker stdout | 2025-11-04 09:22:49,719 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:22:49,719 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:22:49,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 22, 32]) securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 22-32 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 22-32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Left range 22-32 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-37 securityworker stdout | 2025-11-04 09:22:49,721 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 32-37 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 32 securityworker stdout | 2025-11-04 09:22:49,722 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 32, 42]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-47 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 22-32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 22-32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 22-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,724 [92] [DEBUG] [util.migrate.allocator] Right range 22-47 securityworker stdout | 2025-11-04 09:22:49,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stdout | 2025-11-04 09:22:49,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-22 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 12 securityworker stdout | 2025-11-04 09:22:49,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 11, 21]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:22:49,725 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:22:49,725 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:49,726 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 11]) securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-21 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Right range 22-47 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-22 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 21-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 22-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 21-22 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:22:49,727 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 21 securityworker stdout | 2025-11-04 09:22:49,728 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 21, 31]) securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-47 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 22-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:22:49,729 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 47-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:22:49,730 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 47, 49]) securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:22:49,731 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 21, 31]) securityworker stdout | 2025-11-04 09:22:49,732 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stdout | 2025-11-04 09:22:49,732 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stdout | 2025-11-04 09:22:49,732 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 21-31 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 11 securityworker stdout | 2025-11-04 09:22:49,733 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 11, 21]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 11-21 by worker securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-31 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Left range 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 11-21 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 21-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-49 securityworker stdout | 2025-11-04 09:22:49,735 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:22:49,736 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 32, 42]) securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 32-42 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Left range 11-31 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stdout | 2025-11-04 09:22:49,737 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 2 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Left range 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 31-32 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 31 securityworker stdout | 2025-11-04 09:22:49,738 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 31, 41]) securityworker stdout | 2025-11-04 09:22:49,738 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stdout | 2025-11-04 09:22:49,738 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stdout | 2025-11-04 09:22:49,738 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 31-41 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 31-41 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 11-31 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-31 securityworker stdout | 2025-11-04 09:22:49,738 [92] [DEBUG] [util.migrate.allocator] Already merged with block 11-31 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-42 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Right range 11-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 32-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 11-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 11-42 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:49,739 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 1, 11]) securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-42 securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 11-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,740 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,741 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stdout | 2025-11-04 09:22:49,741 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:49,741 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stdout | 2025-11-04 09:22:49,741 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 42-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 42-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 42 securityworker stdout | 2025-11-04 09:22:49,742 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 42, 49]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 42-49 by worker securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 42-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 49-42 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:22:49,743 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 39 securityworker stdout | 2025-11-04 09:22:49,744 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 37, 47]) securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 37-47 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stdout | 2025-11-04 09:22:49,745 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-37 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 27 securityworker stdout | 2025-11-04 09:22:49,746 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 20, 30]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:22:49,747 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 20-30 by worker securityworker stdout | 2025-11-04 09:22:49,747 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Left range 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Left range 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Right range 37-47 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-37 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 30-37 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 30 securityworker stdout | 2025-11-04 09:22:49,748 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 30, 40]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stdout | 2025-11-04 09:22:49,749 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 30-40 by worker securityworker stdout | 2025-11-04 09:22:49,749 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stdout | 2025-11-04 09:22:49,749 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 20-30 securityworker stdout | 2025-11-04 09:22:49,749 [92] [DEBUG] [util.migrate.allocator] Already merged with block 20-30 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-47 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 30-40 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 20-30 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 37-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 2 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 20-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Right range 20-47 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-20 securityworker stdout | 2025-11-04 09:22:49,750 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 10 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 20-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-20 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 10 securityworker stdout | 2025-11-04 09:22:49,751 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 9, 19]) securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 9-19 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 9-19 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Right range 9-19 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stdout | 2025-11-04 09:22:49,752 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Writing new block with range: 9-19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 2 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 3 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Right range 9-19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-9 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:22:49,753 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 1, 11]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-11 by worker securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-19 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 19 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Total range: 19-49 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-11 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with block 9-19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 1 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 19-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 1 with 2 total holes securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Left range 20-47 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stdout | 2025-11-04 09:22:49,754 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Left range 20-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 47-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 47 securityworker stdout | 2025-11-04 09:22:49,755 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 47, 49]) securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 20-47 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Already merged with block 20-47 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 20 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 47-49 by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 47-49 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Merging with the prev range: 20-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Already merged with block 20-47 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 20 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding 1 obsolete blocks securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 19-20 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Total range: 19-20 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-20 securityworker stdout | 2025-11-04 09:22:49,756 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 19-20 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 19 securityworker stdout | 2025-11-04 09:22:49,757 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 17, 49, 710376), 19, 29]) securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 19-29 by worker securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] Total range: 29-19 securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 19-29 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 29 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] Total range: 29-19 securityworker stderr | 2025-11-04 09:22:49 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:22:49,758 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:22:49,758 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:19 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:22:51,346 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,159 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,160 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:00.086435+00:00 (in 7.926290 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,160 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:52 GMT)" (scheduled at 2025-11-04 09:22:52.159714+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,161 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."artifact_type_backfilled" = %s) OR ("t1"."artifact_type_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,167 [73] [DEBUG] [__main__] Manifest artifact_type backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,167 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:22:52,167 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_artifact_type (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:52 GMT)" executed successfully namespacegcworker stdout | 2025-11-04 09:22:53,560 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:22:53,560 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:33.561829+00:00 (in 40.001188 seconds) namespacegcworker stdout | 2025-11-04 09:22:53,560 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:53 GMT)" (scheduled at 2025-11-04 09:22:53.560131+00:00) namespacegcworker stdout | 2025-11-04 09:22:53,560 [75] [DEBUG] [workers.queueworker] Getting work item from queue. namespacegcworker stdout | 2025-11-04 09:22:53,561 [75] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 53, 560979), True, datetime.datetime(2025, 11, 4, 9, 22, 53, 560979), 0, 'namespacegc/%', 50, 1, 0]) namespacegcworker stdout | 2025-11-04 09:22:53,568 [75] [DEBUG] [workers.queueworker] No more work. namespacegcworker stdout | 2025-11-04 09:22:53,568 [75] [DEBUG] [data.database] Disconnecting from database. namespacegcworker stdout | 2025-11-04 09:22:53,568 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:23:53 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:22:55,065 [261] [DEBUG] [app] Starting request: urn:request:17477899-2b69-4174-87e5-b53eb2c926ef (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:22:55,069 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:22:55,072 [260] [DEBUG] [app] Starting request: urn:request:b609a2f9-2711-4a6a-83ed-f2016b140543 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:55,073 [260] [DEBUG] [app] Ending request: urn:request:b609a2f9-2711-4a6a-83ed-f2016b140543 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b609a2f9-2711-4a6a-83ed-f2016b140543', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-registry stdout | 2025-11-04 09:22:55,073 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:55,073 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:55,074 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:55,076 [261] [DEBUG] [app] Starting request: urn:request:298add04-66d6-4096-a9f4-6632a3246871 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:55,076 [261] [DEBUG] [app] Ending request: urn:request:298add04-66d6-4096-a9f4-6632a3246871 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:298add04-66d6-4096-a9f4-6632a3246871', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:22:55,077 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:55,078 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:55,078 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:55,078 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:55,078 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:55,084 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:55,084 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:55,088 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:55,089 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:55,090 [261] [DEBUG] [app] Ending request: urn:request:17477899-2b69-4174-87e5-b53eb2c926ef (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:17477899-2b69-4174-87e5-b53eb2c926ef', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:55,090 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:55,090 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.025 118 0.025) gunicorn-web stdout | 2025-11-04 09:22:55,122 [261] [DEBUG] [app] Starting request: urn:request:2806a16d-f356-483c-a802-930958313304 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:22:55,124 [260] [DEBUG] [app] Starting request: urn:request:b2f69d44-927c-4d21-840c-1a9f4d16ea50 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:22:55,125 [260] [DEBUG] [app] Ending request: urn:request:b2f69d44-927c-4d21-840c-1a9f4d16ea50 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b2f69d44-927c-4d21-840c-1a9f4d16ea50', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:22:55,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:22:55,125 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:22:55,126 [261] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:22:55,128 [254] [DEBUG] [app] Starting request: urn:request:92fca9ed-8ccd-40d1-87e8-a1c4fd6acc43 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:22:55,128 [254] [DEBUG] [app] Ending request: urn:request:92fca9ed-8ccd-40d1-87e8-a1c4fd6acc43 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:92fca9ed-8ccd-40d1-87e8-a1c4fd6acc43', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:22:55,129 [254] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:22:55 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.002) gunicorn-web stdout | 2025-11-04 09:22:55,129 [261] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:22:55,129 [261] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:22:55,129 [261] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:22:55,129 [261] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:22:55,133 [261] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:22:55,133 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:22:55,137 [261] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:22:55,139 [261] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:22:55,139 [261] [DEBUG] [app] Ending request: urn:request:2806a16d-f356-483c-a802-930958313304 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:2806a16d-f356-483c-a802-930958313304', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:22:55,139 [261] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:22:55,140 [261] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:22:55 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:22:55 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.019 118 0.018) gcworker stdout | 2025-11-04 09:22:55,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:22:55,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:07.663502+00:00 (in 12.001416 seconds) gcworker stdout | 2025-11-04 09:22:55,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:25 GMT)" (scheduled at 2025-11-04 09:22:55.661677+00:00) gcworker stdout | 2025-11-04 09:22:55,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038575662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:22:55,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:22:55,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:22:55,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:25 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:22:56,049 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:56,203 [238] [INFO] [gunicorn.access] - - [04/Nov/2025:09:22:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:22:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.001 957 0.001) buildlogsarchiver stdout | 2025-11-04 09:22:56,443 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:22:56,587 [239] [INFO] [gunicorn.access] - - [04/Nov/2025:09:22:56 +0000] "POST /secscan/notification HTTP/1.0" 400 0 "-" "-" nginx stdout | 10.129.4.13 (-) - - [04/Nov/2025:09:22:56 +0000] "POST /secscan/notification HTTP/1.1" 400 171 "-" "clair/v4.8.0 (user) (claircore v1.5.32)" (0.000 957 0.001) notificationworker stdout | 2025-11-04 09:22:57,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:22:57,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:07.766253+00:00 (in 9.999555 seconds) notificationworker stdout | 2025-11-04 09:22:57,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:07 GMT)" (scheduled at 2025-11-04 09:22:57.766253+00:00) notificationworker stdout | 2025-11-04 09:22:57,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:22:57,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 57, 767030), True, datetime.datetime(2025, 11, 4, 9, 22, 57, 767030), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:22:57,774 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:22:57,774 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:22:57,774 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:07 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:22:59,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:22:59,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:09.676299+00:00 (in 9.999577 seconds) proxycacheblobworker stdout | 2025-11-04 09:22:59,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:09 GMT)" (scheduled at 2025-11-04 09:22:59.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:22:59,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:22:59,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 22, 59, 677054), True, datetime.datetime(2025, 11, 4, 9, 22, 59, 677054), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:22:59,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:22:59,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:22:59,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:09 GMT)" executed successfully manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,086 [73] [DEBUG] [apscheduler.scheduler] Looking for jobs to run manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,086 [73] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:52.159714+00:00 (in 52.072856 seconds) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,087 [73] [INFO] [apscheduler.executors.default] Running job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:00 GMT)" (scheduled at 2025-11-04 09:23:00.086435+00:00) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,087 [73] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled" FROM "manifest" AS "t1" WHERE (("t1"."subject_backfilled" = %s) OR ("t1"."subject_backfilled" IS %s)) LIMIT %s OFFSET %s', [False, None, 1, 0]) manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,094 [73] [DEBUG] [__main__] Manifest subject backfill worker has completed; skipping manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,094 [73] [DEBUG] [data.database] Disconnecting from database. manifestsubjectbackfillworker stdout | 2025-11-04 09:23:00,094 [73] [INFO] [apscheduler.executors.default] Job "ManifestSubjectBackfillWorker._backfill_manifest_subject (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:00 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:23:00,254 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:23:00,339 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:00,640 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:23:01,503 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:23:01,638 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:23:01,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:01,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:19.706478+00:00 (in 17.998796 seconds) securityworker stdout | 2025-11-04 09:23:01,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:31 GMT)" (scheduled at 2025-11-04 09:23:01.707296+00:00) securityworker stdout | 2025-11-04 09:23:01,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:01,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:01,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:01,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:01,719 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:01,719 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:01,719 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:23:01,719 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:01,720 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,722 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,722 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:01,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:23:01,723 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:01,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 18, 1, 713745), 1, 49]) securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,724 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,724 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:01,725 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 18, 1, 713745), 1, 49]) securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:01 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:01,727 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:23:01,727 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:31 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:23:02,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:23:02,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:32.081579+00:00 (in 29.999576 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:23:02,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:32 GMT)" (scheduled at 2025-11-04 09:23:02.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:23:02,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:23:02,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:23:02,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:32 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:23:02,825 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: chunkcleanupworker stdout | 2025-11-04 09:23:04,450 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:23:04,541 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:23:05,625 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:23:06,134 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: repositorygcworker stdout | 2025-11-04 09:23:06,266 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:23:06,266 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:40.180734+00:00 (in 33.914053 seconds) repositorygcworker stdout | 2025-11-04 09:23:06,266 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:06 GMT)" (scheduled at 2025-11-04 09:23:06.265965+00:00) repositorygcworker stdout | 2025-11-04 09:23:06,266 [90] [DEBUG] [workers.queueworker] Running watchdog. repositorygcworker stdout | 2025-11-04 09:23:06,266 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:06 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:23:06,569 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:23:06,569 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:33.568037+00:00 (in 26.998095 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:23:06,570 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:06 GMT)" (scheduled at 2025-11-04 09:23:06.569529+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:23:06,570 [91] [DEBUG] [workers.queueworker] Running watchdog. securityscanningnotificationworker stdout | 2025-11-04 09:23:06,570 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:06 GMT)" executed successfully autopruneworker stdout | 2025-11-04 09:23:07,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:23:07,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:37.077358+00:00 (in 29.999556 seconds) autopruneworker stdout | 2025-11-04 09:23:07,077 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:37 GMT)" (scheduled at 2025-11-04 09:23:07.077358+00:00) autopruneworker stdout | 2025-11-04 09:23:07,082 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244587082, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:23:07,085 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:23:07,085 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:23:07,085 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:37 GMT)" executed successfully gcworker stdout | 2025-11-04 09:23:07,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:07,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:25.661677+00:00 (in 17.997759 seconds) gcworker stdout | 2025-11-04 09:23:07,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:37 GMT)" (scheduled at 2025-11-04 09:23:07.663502+00:00) gcworker stdout | 2025-11-04 09:23:07,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:23:07,671 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247887670, None, 1, 0]) gcworker stdout | 2025-11-04 09:23:07,672 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:07,672 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:37 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:07,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:07,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:17.766253+00:00 (in 9.999584 seconds) notificationworker stdout | 2025-11-04 09:23:07,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:17 GMT)" (scheduled at 2025-11-04 09:23:07.766253+00:00) notificationworker stdout | 2025-11-04 09:23:07,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:07,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 7, 767014), True, datetime.datetime(2025, 11, 4, 9, 23, 7, 767014), 0, 'notification/%', 50, 1, 0]) quotaregistrysizeworker stdout | 2025-11-04 09:23:07,770 [82] [DEBUG] [apscheduler.scheduler] Looking for jobs to run quotaregistrysizeworker stdout | 2025-11-04 09:23:07,770 [82] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:07.770322+00:00 (in 59.999571 seconds) quotaregistrysizeworker stdout | 2025-11-04 09:23:07,770 [82] [INFO] [apscheduler.executors.default] Running job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:07 GMT)" (scheduled at 2025-11-04 09:23:07.770322+00:00) quotaregistrysizeworker stdout | 2025-11-04 09:23:07,771 [82] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."size_bytes", "t1"."running", "t1"."queued", "t1"."completed_ms" FROM "quotaregistrysize" AS "t1" LIMIT %s OFFSET %s', [1, 0]) notificationworker stdout | 2025-11-04 09:23:07,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:07,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:07,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:17 GMT)" executed successfully quotaregistrysizeworker stdout | 2025-11-04 09:23:07,776 [82] [DEBUG] [data.database] Disconnecting from database. quotaregistrysizeworker stdout | 2025-11-04 09:23:07,776 [82] [INFO] [apscheduler.executors.default] Job "QuotaRegistrySizeWorker._calculate_registry_size (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:07 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:23:08,629 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:09,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:09,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:19.676299+00:00 (in 9.999535 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:09,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:19 GMT)" (scheduled at 2025-11-04 09:23:09.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:09,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:09,678 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 9, 677134), True, datetime.datetime(2025, 11, 4, 9, 23, 9, 677134), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:09,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:09,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:09,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:19 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:23:09,748 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:10,013 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:10,065 [254] [DEBUG] [app] Starting request: urn:request:d6463535-0665-4c42-b9bc-c8a1877080b1 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:23:10,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:23:10,068 [259] [DEBUG] [app] Starting request: urn:request:ee3d11d2-08a6-47c9-b2a4-b9ff99bb59c2 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:10,069 [259] [DEBUG] [app] Ending request: urn:request:ee3d11d2-08a6-47c9-b2a4-b9ff99bb59c2 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:ee3d11d2-08a6-47c9-b2a4-b9ff99bb59c2', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:10,069 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-web stdout | 2025-11-04 09:23:10,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:10,071 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:10,072 [258] [DEBUG] [app] Starting request: urn:request:c88bba46-0c36-43c7-bad6-ee77bec71a3f (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:10,072 [258] [DEBUG] [app] Ending request: urn:request:c88bba46-0c36-43c7-bad6-ee77bec71a3f (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:c88bba46-0c36-43c7-bad6-ee77bec71a3f', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:10,073 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:10,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:10,073 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:10,073 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:10,073 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:10,077 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:10,077 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:10,081 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:10,082 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:10,083 [254] [DEBUG] [app] Ending request: urn:request:d6463535-0665-4c42-b9bc-c8a1877080b1 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d6463535-0665-4c42-b9bc-c8a1877080b1', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:10,083 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:10,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.019) gunicorn-web stdout | 2025-11-04 09:23:10,122 [254] [DEBUG] [app] Starting request: urn:request:0f250e0a-4cee-4ae3-b2d3-2cbf190293fc (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:23:10,124 [259] [DEBUG] [app] Starting request: urn:request:b4273f17-79ce-47dd-b17f-3fff533aa0c1 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:10,124 [259] [DEBUG] [app] Ending request: urn:request:b4273f17-79ce-47dd-b17f-3fff533aa0c1 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b4273f17-79ce-47dd-b17f-3fff533aa0c1', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.001) gunicorn-registry stdout | 2025-11-04 09:23:10,125 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:10 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:10,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:10,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:10,127 [258] [DEBUG] [app] Starting request: urn:request:3b0d23e7-71ae-4503-8fe6-e0447c0215a0 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:10,127 [258] [DEBUG] [app] Ending request: urn:request:3b0d23e7-71ae-4503-8fe6-e0447c0215a0 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:3b0d23e7-71ae-4503-8fe6-e0447c0215a0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:10,127 [258] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:10 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:10,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:10,128 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:10,128 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:10,128 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:10,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:10,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:10,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:10,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:10,138 [254] [DEBUG] [app] Ending request: urn:request:0f250e0a-4cee-4ae3-b2d3-2cbf190293fc (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:0f250e0a-4cee-4ae3-b2d3-2cbf190293fc', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:10,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:10,139 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:10 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:10 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.017 118 0.017) securityworker stdout | 2025-11-04 09:23:10,424 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:23:10,451 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:23:11,242 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:23:11,526 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:23:11,826 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:12,482 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:23:12,482 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:38.480592+00:00 (in 25.998082 seconds) exportactionlogsworker stdout | 2025-11-04 09:23:12,482 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:12 GMT)" (scheduled at 2025-11-04 09:23:12.482065+00:00) exportactionlogsworker stdout | 2025-11-04 09:23:12,482 [65] [DEBUG] [workers.queueworker] Running watchdog. exportactionlogsworker stdout | 2025-11-04 09:23:12,482 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:12 GMT)" executed successfully gunicorn-secscan stdout | 2025-11-04 09:23:12,532 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:12,549 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:12,555 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:23:12,940 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:13,344 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:23:13,727 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:14,552 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:23:14,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:23:14,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:44.980431+00:00 (in 29.999578 seconds) buildlogsarchiver stdout | 2025-11-04 09:23:14,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:44 GMT)" (scheduled at 2025-11-04 09:23:14.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:23:14,981 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 23, 14, 981129), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:14,988 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:23:14,988 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:23:14,988 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:44 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:15,038 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:15,067 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:15,068 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:15,068 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:15,205 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,563 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,641 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,680 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,724 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,715 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,740 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,745 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,754 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:16,768 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:17,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:17,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:27.766253+00:00 (in 9.999565 seconds) notificationworker stdout | 2025-11-04 09:23:17,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:27 GMT)" (scheduled at 2025-11-04 09:23:17.766253+00:00) notificationworker stdout | 2025-11-04 09:23:17,766 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:17,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 17, 767075), True, datetime.datetime(2025, 11, 4, 9, 23, 17, 767075), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:17,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:17,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:17,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:27 GMT)" executed successfully proxycacheblobworker stdout | 2025-11-04 09:23:19,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:19,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:29.676299+00:00 (in 9.999523 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:19,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:29 GMT)" (scheduled at 2025-11-04 09:23:19.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:19,677 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:19,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 19, 677103), True, datetime.datetime(2025, 11, 4, 9, 23, 19, 677103), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:19,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:19,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:19,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:29 GMT)" executed successfully securityworker stdout | 2025-11-04 09:23:19,706 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:19,706 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:31.707296+00:00 (in 12.000406 seconds) securityworker stdout | 2025-11-04 09:23:19,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:49 GMT)" (scheduled at 2025-11-04 09:23:19.706478+00:00) securityworker stdout | 2025-11-04 09:23:19,707 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:19,707 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:19,708 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:19,710 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stdout | 2025-11-04 09:23:19,716 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stdout | 2025-11-04 09:23:19,717 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:49 GMT)" executed successfully storagereplication stdout | 2025-11-04 09:23:21,358 [96] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'storagereplication.py', 'pid': '96'} storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open storagereplication stdout | h.request(req.get_method(), req.selector, req.data, headers, storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request storagereplication stdout | self._send_request(method, url, body, headers, encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request storagereplication stdout | self.endheaders(body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders storagereplication stdout | self._send_output(message_body, encode_chunked=encode_chunked) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output storagereplication stdout | self.send(msg) storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send storagereplication stdout | self.connect() storagereplication stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect storagereplication stdout | self.sock = self._create_connection( storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection storagereplication stdout | raise exceptions[0] storagereplication stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection storagereplication stdout | sock.connect(sa) storagereplication stdout | ConnectionRefusedError: [Errno 111] Connection refused storagereplication stdout | During handling of the above exception, another exception occurred: storagereplication stdout | Traceback (most recent call last): storagereplication stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run storagereplication stdout | push_to_gateway( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway storagereplication stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway storagereplication stdout | handler( storagereplication stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle storagereplication stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open storagereplication stdout | response = self._open(req, data) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open storagereplication stdout | result = self._call_chain(self.handle_open, protocol, protocol + storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain storagereplication stdout | result = func(*args) storagereplication stdout | ^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open storagereplication stdout | return self.do_open(http.client.HTTPConnection, req) storagereplication stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ storagereplication stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open storagereplication stdout | raise URLError(err) storagereplication stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:25,065 [254] [DEBUG] [app] Starting request: urn:request:d7d47e68-e174-4714-855d-ff43e72b7e5b (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:23:25,067 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:23:25,068 [259] [DEBUG] [app] Starting request: urn:request:b5707a3d-1526-40b0-954f-3789530a0b08 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:25,069 [259] [DEBUG] [app] Ending request: urn:request:b5707a3d-1526-40b0-954f-3789530a0b08 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:b5707a3d-1526-40b0-954f-3789530a0b08', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:25,069 [259] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:25,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:23:25,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:25,074 [261] [DEBUG] [app] Starting request: urn:request:fabaf3fa-7a8f-4370-ab5d-0fd0fe175d2e (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:25,075 [261] [DEBUG] [app] Ending request: urn:request:fabaf3fa-7a8f-4370-ab5d-0fd0fe175d2e (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:fabaf3fa-7a8f-4370-ab5d-0fd0fe175d2e', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:23:25,075 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:25,075 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:25,076 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:25,076 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:25,076 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:25,080 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:25,080 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:25,085 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:25,086 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:25,087 [254] [DEBUG] [app] Ending request: urn:request:d7d47e68-e174-4714-855d-ff43e72b7e5b (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:d7d47e68-e174-4714-855d-ff43e72b7e5b', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:25,087 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:25,088 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.023 118 0.023) gunicorn-web stdout | 2025-11-04 09:23:25,123 [254] [DEBUG] [app] Starting request: urn:request:3660d9e7-387c-44d6-ab87-7082732d9719 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:23:25,124 [260] [DEBUG] [app] Starting request: urn:request:892a5053-c348-4e10-8da4-da3e6dacbebb (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:25,125 [260] [DEBUG] [app] Ending request: urn:request:892a5053-c348-4e10-8da4-da3e6dacbebb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:892a5053-c348-4e10-8da4-da3e6dacbebb', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:25,125 [260] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:25,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.002) gunicorn-web stdout | 2025-11-04 09:23:25,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:25,127 [261] [DEBUG] [app] Starting request: urn:request:6b9a96f6-d217-4255-9478-d51f692e43d6 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:25,128 [261] [DEBUG] [app] Ending request: urn:request:6b9a96f6-d217-4255-9478-d51f692e43d6 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:6b9a96f6-d217-4255-9478-d51f692e43d6', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:25,128 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:25 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:25,129 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.002) gunicorn-web stdout | 2025-11-04 09:23:25,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:25,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:25,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:25,133 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:25,133 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:25,137 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:25,138 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:25,138 [254] [DEBUG] [app] Ending request: urn:request:3660d9e7-387c-44d6-ab87-7082732d9719 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:3660d9e7-387c-44d6-ab87-7082732d9719', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:25,138 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:25,138 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:25 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:25 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.016 118 0.017) gcworker stdout | 2025-11-04 09:23:25,661 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:25,662 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:37.663502+00:00 (in 12.001407 seconds) gcworker stdout | 2025-11-04 09:23:25,662 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:55 GMT)" (scheduled at 2025-11-04 09:23:25.661677+00:00) gcworker stdout | 2025-11-04 09:23:25,663 [66] [DEBUG] [peewee] ('SELECT "candidates"."repository_id" FROM (SELECT DISTINCT "t1"."repository_id" FROM "tag" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") INNER JOIN "user" AS "t3" ON ("t2"."namespace_user_id" = "t3"."id") WHERE ((((NOT ("t1"."lifetime_end_ms" IS %s) AND ("t1"."lifetime_end_ms" <= %s)) AND ("t3"."removed_tag_expiration_s" = %s)) AND ("t3"."enabled" = %s)) AND ("t2"."state" != %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', [None, 1761038605662, 1209600, True, 3, 500, 1, 0]) gcworker stdout | 2025-11-04 09:23:25,671 [66] [DEBUG] [__main__] No repository with garbage found gcworker stdout | 2025-11-04 09:23:25,671 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:25,671 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._garbage_collection_repos (trigger: interval[0:00:30], next run at: 2025-11-04 09:23:55 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:26,062 [77] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'notificationworker.py', 'pid': '77'} notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open notificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request notificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request notificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders notificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output notificationworker stdout | self.send(msg) notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send notificationworker stdout | self.connect() notificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect notificationworker stdout | self.sock = self._create_connection( notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection notificationworker stdout | raise exceptions[0] notificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection notificationworker stdout | sock.connect(sa) notificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused notificationworker stdout | During handling of the above exception, another exception occurred: notificationworker stdout | Traceback (most recent call last): notificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run notificationworker stdout | push_to_gateway( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway notificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway notificationworker stdout | handler( notificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle notificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open notificationworker stdout | response = self._open(req, data) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open notificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain notificationworker stdout | result = func(*args) notificationworker stdout | ^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open notificationworker stdout | return self.do_open(http.client.HTTPConnection, req) notificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ notificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open notificationworker stdout | raise URLError(err) notificationworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:23:26,456 [61] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'buildlogsarchiver.py', 'pid': '61'} buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open buildlogsarchiver stdout | h.request(req.get_method(), req.selector, req.data, headers, buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request buildlogsarchiver stdout | self._send_request(method, url, body, headers, encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request buildlogsarchiver stdout | self.endheaders(body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders buildlogsarchiver stdout | self._send_output(message_body, encode_chunked=encode_chunked) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output buildlogsarchiver stdout | self.send(msg) buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send buildlogsarchiver stdout | self.connect() buildlogsarchiver stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect buildlogsarchiver stdout | self.sock = self._create_connection( buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection buildlogsarchiver stdout | raise exceptions[0] buildlogsarchiver stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection buildlogsarchiver stdout | sock.connect(sa) buildlogsarchiver stdout | ConnectionRefusedError: [Errno 111] Connection refused buildlogsarchiver stdout | During handling of the above exception, another exception occurred: buildlogsarchiver stdout | Traceback (most recent call last): buildlogsarchiver stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run buildlogsarchiver stdout | push_to_gateway( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway buildlogsarchiver stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway buildlogsarchiver stdout | handler( buildlogsarchiver stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle buildlogsarchiver stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open buildlogsarchiver stdout | response = self._open(req, data) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open buildlogsarchiver stdout | result = self._call_chain(self.handle_open, protocol, protocol + buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain buildlogsarchiver stdout | result = func(*args) buildlogsarchiver stdout | ^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open buildlogsarchiver stdout | return self.do_open(http.client.HTTPConnection, req) buildlogsarchiver stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ buildlogsarchiver stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open buildlogsarchiver stdout | raise URLError(err) buildlogsarchiver stdout | urllib.error.URLError: notificationworker stdout | 2025-11-04 09:23:27,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:27,767 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:27.768179+00:00 (in 0.000993 seconds) notificationworker stdout | 2025-11-04 09:23:27,767 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:37 GMT)" (scheduled at 2025-11-04 09:23:27.766253+00:00) notificationworker stdout | 2025-11-04 09:23:27,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:27,768 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 27, 767584), True, datetime.datetime(2025, 11, 4, 9, 23, 27, 767584), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:27,768 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:27,768 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:37.766253+00:00 (in 9.997620 seconds) notificationworker stdout | 2025-11-04 09:23:27,768 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:27 GMT)" (scheduled at 2025-11-04 09:23:27.768179+00:00) notificationworker stdout | 2025-11-04 09:23:27,768 [77] [DEBUG] [workers.queueworker] Running watchdog. notificationworker stdout | 2025-11-04 09:23:27,768 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:27 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:27,776 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:27,776 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:27,776 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:37 GMT)" executed successfully quotatotalworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotatotalworker.py', 'pid': '83'} quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotatotalworker stderr | h.request(req.get_method(), req.selector, req.data, headers, quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotatotalworker stderr | self._send_request(method, url, body, headers, encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotatotalworker stderr | self.endheaders(body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotatotalworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotatotalworker stderr | self.send(msg) quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotatotalworker stderr | self.connect() quotatotalworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotatotalworker stderr | self.sock = self._create_connection( quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotatotalworker stderr | raise exceptions[0] quotatotalworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotatotalworker stderr | sock.connect(sa) quotatotalworker stderr | ConnectionRefusedError: [Errno 111] Connection refused quotatotalworker stderr | During handling of the above exception, another exception occurred: quotatotalworker stderr | Traceback (most recent call last): quotatotalworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotatotalworker stderr | push_to_gateway( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotatotalworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotatotalworker stderr | handler( quotatotalworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotatotalworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotatotalworker stderr | response = self._open(req, data) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotatotalworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotatotalworker stderr | result = func(*args) quotatotalworker stderr | ^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotatotalworker stderr | return self.do_open(http.client.HTTPConnection, req) quotatotalworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotatotalworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotatotalworker stderr | raise URLError(err) quotatotalworker stderr | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:29,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:29,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:39.676299+00:00 (in 9.999581 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:29,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:39 GMT)" (scheduled at 2025-11-04 09:23:29.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:29,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:29,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 29, 677044), True, datetime.datetime(2025, 11, 4, 9, 23, 29, 677044), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:29,685 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:29,685 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:29,685 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:39 GMT)" executed successfully repositorygcworker stdout | 2025-11-04 09:23:30,266 [90] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositorygcworker.py', 'pid': '90'} repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositorygcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositorygcworker stdout | self._send_request(method, url, body, headers, encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositorygcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositorygcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositorygcworker stdout | self.send(msg) repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositorygcworker stdout | self.connect() repositorygcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositorygcworker stdout | self.sock = self._create_connection( repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositorygcworker stdout | raise exceptions[0] repositorygcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositorygcworker stdout | sock.connect(sa) repositorygcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused repositorygcworker stdout | During handling of the above exception, another exception occurred: repositorygcworker stdout | Traceback (most recent call last): repositorygcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositorygcworker stdout | push_to_gateway( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositorygcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositorygcworker stdout | handler( repositorygcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositorygcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositorygcworker stdout | response = self._open(req, data) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositorygcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositorygcworker stdout | result = func(*args) repositorygcworker stdout | ^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositorygcworker stdout | return self.do_open(http.client.HTTPConnection, req) repositorygcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositorygcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositorygcworker stdout | raise URLError(err) repositorygcworker stdout | urllib.error.URLError: globalpromstats stdout | 2025-11-04 09:23:30,353 [67] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'globalpromstats.py', 'pid': '67'} globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open globalpromstats stdout | h.request(req.get_method(), req.selector, req.data, headers, globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request globalpromstats stdout | self._send_request(method, url, body, headers, encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request globalpromstats stdout | self.endheaders(body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders globalpromstats stdout | self._send_output(message_body, encode_chunked=encode_chunked) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output globalpromstats stdout | self.send(msg) globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send globalpromstats stdout | self.connect() globalpromstats stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect globalpromstats stdout | self.sock = self._create_connection( globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection globalpromstats stdout | raise exceptions[0] globalpromstats stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection globalpromstats stdout | sock.connect(sa) globalpromstats stdout | ConnectionRefusedError: [Errno 111] Connection refused globalpromstats stdout | During handling of the above exception, another exception occurred: globalpromstats stdout | Traceback (most recent call last): globalpromstats stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run globalpromstats stdout | push_to_gateway( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway globalpromstats stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway globalpromstats stdout | handler( globalpromstats stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle globalpromstats stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open globalpromstats stdout | response = self._open(req, data) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open globalpromstats stdout | result = self._call_chain(self.handle_open, protocol, protocol + globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain globalpromstats stdout | result = func(*args) globalpromstats stdout | ^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open globalpromstats stdout | return self.do_open(http.client.HTTPConnection, req) globalpromstats stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ globalpromstats stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open globalpromstats stdout | raise URLError(err) globalpromstats stdout | urllib.error.URLError: builder stdout | 2025-11-04 09:23:30,652 [60] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'builder.py', 'pid': '60'} builder stdout | Traceback (most recent call last): builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open builder stdout | h.request(req.get_method(), req.selector, req.data, headers, builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request builder stdout | self._send_request(method, url, body, headers, encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request builder stdout | self.endheaders(body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders builder stdout | self._send_output(message_body, encode_chunked=encode_chunked) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output builder stdout | self.send(msg) builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send builder stdout | self.connect() builder stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect builder stdout | self.sock = self._create_connection( builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection builder stdout | raise exceptions[0] builder stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection builder stdout | sock.connect(sa) builder stdout | ConnectionRefusedError: [Errno 111] Connection refused builder stdout | During handling of the above exception, another exception occurred: builder stdout | Traceback (most recent call last): builder stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run builder stdout | push_to_gateway( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway builder stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway builder stdout | handler( builder stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle builder stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open builder stdout | response = self._open(req, data) builder stdout | ^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open builder stdout | result = self._call_chain(self.handle_open, protocol, protocol + builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain builder stdout | result = func(*args) builder stdout | ^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open builder stdout | return self.do_open(http.client.HTTPConnection, req) builder stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ builder stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open builder stdout | raise URLError(err) builder stdout | urllib.error.URLError: repositoryactioncounter stdout | 2025-11-04 09:23:31,524 [85] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'repositoryactioncounter.py', 'pid': '85'} repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open repositoryactioncounter stdout | h.request(req.get_method(), req.selector, req.data, headers, repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request repositoryactioncounter stdout | self._send_request(method, url, body, headers, encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request repositoryactioncounter stdout | self.endheaders(body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders repositoryactioncounter stdout | self._send_output(message_body, encode_chunked=encode_chunked) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output repositoryactioncounter stdout | self.send(msg) repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send repositoryactioncounter stdout | self.connect() repositoryactioncounter stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect repositoryactioncounter stdout | self.sock = self._create_connection( repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection repositoryactioncounter stdout | raise exceptions[0] repositoryactioncounter stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection repositoryactioncounter stdout | sock.connect(sa) repositoryactioncounter stdout | ConnectionRefusedError: [Errno 111] Connection refused repositoryactioncounter stdout | During handling of the above exception, another exception occurred: repositoryactioncounter stdout | Traceback (most recent call last): repositoryactioncounter stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run repositoryactioncounter stdout | push_to_gateway( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway repositoryactioncounter stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway repositoryactioncounter stdout | handler( repositoryactioncounter stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle repositoryactioncounter stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open repositoryactioncounter stdout | response = self._open(req, data) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open repositoryactioncounter stdout | result = self._call_chain(self.handle_open, protocol, protocol + repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain repositoryactioncounter stdout | result = func(*args) repositoryactioncounter stdout | ^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open repositoryactioncounter stdout | return self.do_open(http.client.HTTPConnection, req) repositoryactioncounter stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ repositoryactioncounter stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open repositoryactioncounter stdout | raise URLError(err) repositoryactioncounter stdout | urllib.error.URLError: reconciliationworker stderr | failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'reconciliationworker.py', 'pid': '84'} reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open reconciliationworker stderr | h.request(req.get_method(), req.selector, req.data, headers, reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1338, in request reconciliationworker stderr | self._send_request(method, url, body, headers, encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request reconciliationworker stderr | self.endheaders(body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders reconciliationworker stderr | self._send_output(message_body, encode_chunked=encode_chunked) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output reconciliationworker stderr | self.send(msg) reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1037, in send reconciliationworker stderr | self.connect() reconciliationworker stderr | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect reconciliationworker stderr | self.sock = self._create_connection( reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection reconciliationworker stderr | raise exceptions[0] reconciliationworker stderr | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection reconciliationworker stderr | sock.connect(sa) reconciliationworker stderr | ConnectionRefusedError: [Errno 111] Connection refused reconciliationworker stderr | During handling of the above exception, another exception occurred: reconciliationworker stderr | Traceback (most recent call last): reconciliationworker stderr | File "/quay-registry/util/metrics/prometheus.py", line 140, in run reconciliationworker stderr | push_to_gateway( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway reconciliationworker stderr | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway reconciliationworker stderr | handler( reconciliationworker stderr | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle reconciliationworker stderr | resp = build_opener(HTTPHandler).open(request, timeout=timeout) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open reconciliationworker stderr | response = self._open(req, data) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open reconciliationworker stderr | result = self._call_chain(self.handle_open, protocol, protocol + reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain reconciliationworker stderr | result = func(*args) reconciliationworker stderr | ^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open reconciliationworker stderr | return self.do_open(http.client.HTTPConnection, req) reconciliationworker stderr | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ reconciliationworker stderr | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open reconciliationworker stderr | raise URLError(err) reconciliationworker stderr | urllib.error.URLError: manifestsubjectbackfillworker stdout | 2025-11-04 09:23:31,651 [73] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestsubjectbackfillworker.py', 'pid': '73'} manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestsubjectbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestsubjectbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestsubjectbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestsubjectbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestsubjectbackfillworker stdout | self.send(msg) manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestsubjectbackfillworker stdout | self.connect() manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestsubjectbackfillworker stdout | self.sock = self._create_connection( manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestsubjectbackfillworker stdout | raise exceptions[0] manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestsubjectbackfillworker stdout | sock.connect(sa) manifestsubjectbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestsubjectbackfillworker stdout | During handling of the above exception, another exception occurred: manifestsubjectbackfillworker stdout | Traceback (most recent call last): manifestsubjectbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestsubjectbackfillworker stdout | push_to_gateway( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestsubjectbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestsubjectbackfillworker stdout | handler( manifestsubjectbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestsubjectbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestsubjectbackfillworker stdout | response = self._open(req, data) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestsubjectbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestsubjectbackfillworker stdout | result = func(*args) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestsubjectbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestsubjectbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestsubjectbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestsubjectbackfillworker stdout | raise URLError(err) manifestsubjectbackfillworker stdout | urllib.error.URLError: securityworker stdout | 2025-11-04 09:23:31,707 [92] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityworker stdout | 2025-11-04 09:23:31,707 [92] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:49.706478+00:00 (in 17.998734 seconds) securityworker stdout | 2025-11-04 09:23:31,707 [92] [INFO] [apscheduler.executors.default] Running job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:01 GMT)" (scheduled at 2025-11-04 09:23:31.707296+00:00) securityworker stdout | 2025-11-04 09:23:31,708 [92] [DEBUG] [util.secscan.v4.api] generated jwt for security scanner request securityworker stdout | 2025-11-04 09:23:31,708 [92] [DEBUG] [util.secscan.v4.api] GETing security URL http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local/indexer/api/v1/index_state securityworker stdout | 2025-11-04 09:23:31,711 [92] [DEBUG] [urllib3.connectionpool] http://quayregistry-clair-app.quay-enterprise-15509.svc.cluster.local:80 "GET /indexer/api/v1/index_state HTTP/1.1" 200 None securityworker stdout | 2025-11-04 09:23:31,713 [92] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "manifest" AS "t1"', []) securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:31,718 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:31,718 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:31,718 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:23:31,718 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:31,719 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" LEFT OUTER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE ((("t2"."id" IS %s) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [None, 1, 49]) securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:31,721 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:31,721 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:23:31,721 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:31,722 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:31,723 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((("t2"."index_status" = %s) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-1, datetime.datetime(2025, 11, 4, 9, 18, 31, 713434), 1, 49]) securityworker stdout | 2025-11-04 09:23:31,723 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:31,723 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:23:31,723 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selected random hole 0 with 1 total holes securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Selecting from hole range: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Rand max bound: 1 securityworker stdout | 2025-11-04 09:23:31,724 [92] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."digest", "t1"."media_type_id", "t1"."manifest_bytes", "t1"."config_media_type", "t1"."layers_compressed_size", "t1"."subject", "t1"."subject_backfilled", "t1"."artifact_type", "t1"."artifact_type_backfilled", "t2"."id", "t2"."manifest_id", "t2"."repository_id", "t2"."index_status", "t2"."error_json", "t2"."last_indexed", "t2"."indexer_hash", "t2"."indexer_version", "t2"."metadata_json" FROM "manifest" AS "t1" INNER JOIN "manifestsecuritystatus" AS "t2" ON ("t2"."manifest_id" = "t1"."id") WHERE (((((("t2"."index_status" != %s) AND ("t2"."index_status" != %s)) AND ("t2"."indexer_hash" != %s)) AND ("t2"."last_indexed" < %s)) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [-2, -3, '37b46b4a70b6f1a19d5e4e18d21f57ff', datetime.datetime(2025, 11, 4, 9, 18, 31, 713434), 1, 49]) securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] No candidates, marking entire block completed 1-49 by worker securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:31,726 [92] [DEBUG] [data.database] Disconnecting from database. securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Marking the range completed: 1-49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new max to: 1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Discarding block and setting new min to: 49 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total blocks: 0 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] Total range: 49-1 securityworker stderr | 2025-11-04 09:23:31 [92] [DEBUG] [util.migrate.allocator] No more work by worker securityworker stdout | 2025-11-04 09:23:31,726 [92] [INFO] [apscheduler.executors.default] Job "SecurityWorker._index_recent_manifests_in_scanner (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:01 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:23:32,081 [79] [DEBUG] [apscheduler.scheduler] Looking for jobs to run pullstatsredisflushworker stdout | 2025-11-04 09:23:32,082 [79] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:02.081579+00:00 (in 29.999582 seconds) pullstatsredisflushworker stdout | 2025-11-04 09:23:32,082 [79] [INFO] [apscheduler.executors.default] Running job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:02 GMT)" (scheduled at 2025-11-04 09:23:32.081579+00:00) pullstatsredisflushworker stdout | 2025-11-04 09:23:32,082 [79] [DEBUG] [__main__] RedisFlushWorker: Starting pull metrics flush pullstatsredisflushworker stdout | 2025-11-04 09:23:32,085 [79] [DEBUG] [__main__] RedisFlushWorker: No pull event keys found pullstatsredisflushworker stdout | 2025-11-04 09:23:32,085 [79] [INFO] [apscheduler.executors.default] Job "RedisFlushWorker._flush_pull_metrics (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:02 GMT)" executed successfully expiredappspecifictokenworker stdout | 2025-11-04 09:23:32,851 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'expiredappspecifictokenworker.py', 'pid': '64'} expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open expiredappspecifictokenworker stdout | h.request(req.get_method(), req.selector, req.data, headers, expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request expiredappspecifictokenworker stdout | self._send_request(method, url, body, headers, encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request expiredappspecifictokenworker stdout | self.endheaders(body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders expiredappspecifictokenworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output expiredappspecifictokenworker stdout | self.send(msg) expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send expiredappspecifictokenworker stdout | self.connect() expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect expiredappspecifictokenworker stdout | self.sock = self._create_connection( expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection expiredappspecifictokenworker stdout | raise exceptions[0] expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection expiredappspecifictokenworker stdout | sock.connect(sa) expiredappspecifictokenworker stdout | ConnectionRefusedError: [Errno 111] Connection refused expiredappspecifictokenworker stdout | During handling of the above exception, another exception occurred: expiredappspecifictokenworker stdout | Traceback (most recent call last): expiredappspecifictokenworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run expiredappspecifictokenworker stdout | push_to_gateway( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway expiredappspecifictokenworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway expiredappspecifictokenworker stdout | handler( expiredappspecifictokenworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle expiredappspecifictokenworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open expiredappspecifictokenworker stdout | response = self._open(req, data) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open expiredappspecifictokenworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain expiredappspecifictokenworker stdout | result = func(*args) expiredappspecifictokenworker stdout | ^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open expiredappspecifictokenworker stdout | return self.do_open(http.client.HTTPConnection, req) expiredappspecifictokenworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expiredappspecifictokenworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open expiredappspecifictokenworker stdout | raise URLError(err) expiredappspecifictokenworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:33,562 [75] [DEBUG] [apscheduler.scheduler] Looking for jobs to run namespacegcworker stdout | 2025-11-04 09:23:33,562 [75] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:53.560131+00:00 (in 19.997786 seconds) namespacegcworker stdout | 2025-11-04 09:23:33,562 [75] [INFO] [apscheduler.executors.default] Running job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" (scheduled at 2025-11-04 09:23:33.561829+00:00) namespacegcworker stdout | 2025-11-04 09:23:33,562 [75] [DEBUG] [workers.queueworker] Running watchdog. namespacegcworker stdout | 2025-11-04 09:23:33,562 [75] [INFO] [apscheduler.executors.default] Job "QueueWorker.run_watchdog (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" executed successfully securityscanningnotificationworker stdout | 2025-11-04 09:23:33,568 [91] [DEBUG] [apscheduler.scheduler] Looking for jobs to run securityscanningnotificationworker stdout | 2025-11-04 09:23:33,568 [91] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:06.569529+00:00 (in 33.001068 seconds) securityscanningnotificationworker stdout | 2025-11-04 09:23:33,568 [91] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" (scheduled at 2025-11-04 09:23:33.568037+00:00) securityscanningnotificationworker stdout | 2025-11-04 09:23:33,568 [91] [DEBUG] [workers.queueworker] Getting work item from queue. securityscanningnotificationworker stdout | 2025-11-04 09:23:33,569 [91] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 33, 568733), True, datetime.datetime(2025, 11, 4, 9, 23, 33, 568733), 0, 'secscanv4/%', 50, 1, 0]) securityscanningnotificationworker stdout | 2025-11-04 09:23:33,576 [91] [DEBUG] [workers.queueworker] No more work. securityscanningnotificationworker stdout | 2025-11-04 09:23:33,576 [91] [DEBUG] [data.database] Disconnecting from database. securityscanningnotificationworker stdout | 2025-11-04 09:23:33,576 [91] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:33 GMT)" executed successfully chunkcleanupworker stdout | 2025-11-04 09:23:34,463 [62] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'chunkcleanupworker.py', 'pid': '62'} chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open chunkcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request chunkcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request chunkcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders chunkcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output chunkcleanupworker stdout | self.send(msg) chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send chunkcleanupworker stdout | self.connect() chunkcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect chunkcleanupworker stdout | self.sock = self._create_connection( chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection chunkcleanupworker stdout | raise exceptions[0] chunkcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection chunkcleanupworker stdout | sock.connect(sa) chunkcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused chunkcleanupworker stdout | During handling of the above exception, another exception occurred: chunkcleanupworker stdout | Traceback (most recent call last): chunkcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run chunkcleanupworker stdout | push_to_gateway( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway chunkcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway chunkcleanupworker stdout | handler( chunkcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle chunkcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open chunkcleanupworker stdout | response = self._open(req, data) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open chunkcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain chunkcleanupworker stdout | result = func(*args) chunkcleanupworker stdout | ^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open chunkcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) chunkcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ chunkcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open chunkcleanupworker stdout | raise URLError(err) chunkcleanupworker stdout | urllib.error.URLError: queuecleanupworker stdout | 2025-11-04 09:23:34,553 [81] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'queuecleanupworker.py', 'pid': '81'} queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open queuecleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request queuecleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request queuecleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders queuecleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output queuecleanupworker stdout | self.send(msg) queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send queuecleanupworker stdout | self.connect() queuecleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect queuecleanupworker stdout | self.sock = self._create_connection( queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection queuecleanupworker stdout | raise exceptions[0] queuecleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection queuecleanupworker stdout | sock.connect(sa) queuecleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused queuecleanupworker stdout | During handling of the above exception, another exception occurred: queuecleanupworker stdout | Traceback (most recent call last): queuecleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run queuecleanupworker stdout | push_to_gateway( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway queuecleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway queuecleanupworker stdout | handler( queuecleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle queuecleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open queuecleanupworker stdout | response = self._open(req, data) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open queuecleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain queuecleanupworker stdout | result = func(*args) queuecleanupworker stdout | ^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open queuecleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) queuecleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ queuecleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open queuecleanupworker stdout | raise URLError(err) queuecleanupworker stdout | urllib.error.URLError: securityscanningnotificationworker stdout | 2025-11-04 09:23:35,637 [91] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityscanningnotificationworker.py', 'pid': '91'} securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityscanningnotificationworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityscanningnotificationworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityscanningnotificationworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityscanningnotificationworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityscanningnotificationworker stdout | self.send(msg) securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityscanningnotificationworker stdout | self.connect() securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityscanningnotificationworker stdout | self.sock = self._create_connection( securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityscanningnotificationworker stdout | raise exceptions[0] securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityscanningnotificationworker stdout | sock.connect(sa) securityscanningnotificationworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityscanningnotificationworker stdout | During handling of the above exception, another exception occurred: securityscanningnotificationworker stdout | Traceback (most recent call last): securityscanningnotificationworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityscanningnotificationworker stdout | push_to_gateway( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityscanningnotificationworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityscanningnotificationworker stdout | handler( securityscanningnotificationworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityscanningnotificationworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityscanningnotificationworker stdout | response = self._open(req, data) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityscanningnotificationworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityscanningnotificationworker stdout | result = func(*args) securityscanningnotificationworker stdout | ^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityscanningnotificationworker stdout | return self.do_open(http.client.HTTPConnection, req) securityscanningnotificationworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityscanningnotificationworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityscanningnotificationworker stdout | raise URLError(err) securityscanningnotificationworker stdout | urllib.error.URLError: teamsyncworker stdout | 2025-11-04 09:23:36,146 [99] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'teamsyncworker.py', 'pid': '99'} teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open teamsyncworker stdout | h.request(req.get_method(), req.selector, req.data, headers, teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request teamsyncworker stdout | self._send_request(method, url, body, headers, encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request teamsyncworker stdout | self.endheaders(body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders teamsyncworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output teamsyncworker stdout | self.send(msg) teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send teamsyncworker stdout | self.connect() teamsyncworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect teamsyncworker stdout | self.sock = self._create_connection( teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection teamsyncworker stdout | raise exceptions[0] teamsyncworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection teamsyncworker stdout | sock.connect(sa) teamsyncworker stdout | ConnectionRefusedError: [Errno 111] Connection refused teamsyncworker stdout | During handling of the above exception, another exception occurred: teamsyncworker stdout | Traceback (most recent call last): teamsyncworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run teamsyncworker stdout | push_to_gateway( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway teamsyncworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway teamsyncworker stdout | handler( teamsyncworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle teamsyncworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open teamsyncworker stdout | response = self._open(req, data) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open teamsyncworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain teamsyncworker stdout | result = func(*args) teamsyncworker stdout | ^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open teamsyncworker stdout | return self.do_open(http.client.HTTPConnection, req) teamsyncworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ teamsyncworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open teamsyncworker stdout | raise URLError(err) teamsyncworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:23:37,077 [58] [DEBUG] [apscheduler.scheduler] Looking for jobs to run autopruneworker stdout | 2025-11-04 09:23:37,077 [58] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:07.077358+00:00 (in 29.999537 seconds) autopruneworker stdout | 2025-11-04 09:23:37,078 [58] [INFO] [apscheduler.executors.default] Running job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:07 GMT)" (scheduled at 2025-11-04 09:23:37.077358+00:00) autopruneworker stdout | 2025-11-04 09:23:37,083 [58] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."namespace_id", "t1"."last_ran_ms", "t1"."status" FROM "autoprunetaskstatus" AS "t1" WHERE (("t1"."namespace_id" NOT IN (SELECT "t2"."id" FROM "user" AS "t2" WHERE (("t2"."enabled" = %s) AND ("t2"."id" = "t1"."namespace_id")))) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [False, 1762244617083, None, 1, 0]) autopruneworker stdout | 2025-11-04 09:23:37,086 [58] [INFO] [__main__] no autoprune tasks found, exiting... autopruneworker stdout | 2025-11-04 09:23:37,086 [58] [DEBUG] [data.database] Disconnecting from database. autopruneworker stdout | 2025-11-04 09:23:37,086 [58] [INFO] [apscheduler.executors.default] Job "AutoPruneWorker.prune (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:07 GMT)" executed successfully gcworker stdout | 2025-11-04 09:23:37,663 [66] [DEBUG] [apscheduler.scheduler] Looking for jobs to run gcworker stdout | 2025-11-04 09:23:37,663 [66] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:55.661677+00:00 (in 17.997749 seconds) gcworker stdout | 2025-11-04 09:23:37,664 [66] [INFO] [apscheduler.executors.default] Running job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:07 GMT)" (scheduled at 2025-11-04 09:23:37.663502+00:00) gcworker stdout | 2025-11-04 09:23:37,664 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "externalnotificationevent" AS "t1" WHERE ("t1"."name" = %s) LIMIT %s OFFSET %s', ['repo_image_expiry', 1, 0]) gcworker stdout | 2025-11-04 09:23:37,672 [66] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."uuid", "t1"."method_id", "t1"."repository_id", "t1"."event_config_json" FROM "repositorynotification" AS "t1" WHERE ((("t1"."event_id" = %s) AND ("t1"."number_of_failures" < %s)) AND (("t1"."last_ran_ms" < %s) OR ("t1"."last_ran_ms" IS %s))) ORDER BY "t1"."last_ran_ms" ASC NULLS first LIMIT %s OFFSET %s FOR UPDATE SKIP LOCKED', [11, 3, 1762247917672, None, 1, 0]) gcworker stdout | 2025-11-04 09:23:37,673 [66] [DEBUG] [data.database] Disconnecting from database. gcworker stdout | 2025-11-04 09:23:37,674 [66] [INFO] [apscheduler.executors.default] Job "GarbageCollectionWorker._scan_notifications (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:07 GMT)" executed successfully notificationworker stdout | 2025-11-04 09:23:37,766 [77] [DEBUG] [apscheduler.scheduler] Looking for jobs to run notificationworker stdout | 2025-11-04 09:23:37,766 [77] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:47.766253+00:00 (in 9.999541 seconds) notificationworker stdout | 2025-11-04 09:23:37,766 [77] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:47 GMT)" (scheduled at 2025-11-04 09:23:37.766253+00:00) notificationworker stdout | 2025-11-04 09:23:37,767 [77] [DEBUG] [workers.queueworker] Getting work item from queue. notificationworker stdout | 2025-11-04 09:23:37,767 [77] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 37, 767070), True, datetime.datetime(2025, 11, 4, 9, 23, 37, 767070), 0, 'notification/%', 50, 1, 0]) notificationworker stdout | 2025-11-04 09:23:37,775 [77] [DEBUG] [workers.queueworker] No more work. notificationworker stdout | 2025-11-04 09:23:37,775 [77] [DEBUG] [data.database] Disconnecting from database. notificationworker stdout | 2025-11-04 09:23:37,775 [77] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:47 GMT)" executed successfully exportactionlogsworker stdout | 2025-11-04 09:23:38,480 [65] [DEBUG] [apscheduler.scheduler] Looking for jobs to run exportactionlogsworker stdout | 2025-11-04 09:23:38,481 [65] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:12.482065+00:00 (in 34.001029 seconds) exportactionlogsworker stdout | 2025-11-04 09:23:38,481 [65] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:38 GMT)" (scheduled at 2025-11-04 09:23:38.480592+00:00) exportactionlogsworker stdout | 2025-11-04 09:23:38,481 [65] [DEBUG] [workers.queueworker] Getting work item from queue. exportactionlogsworker stdout | 2025-11-04 09:23:38,482 [65] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 38, 481304), True, datetime.datetime(2025, 11, 4, 9, 23, 38, 481304), 0, 'exportactionlogs/%', 50, 1, 0]) exportactionlogsworker stdout | 2025-11-04 09:23:38,489 [65] [DEBUG] [workers.queueworker] No more work. exportactionlogsworker stdout | 2025-11-04 09:23:38,489 [65] [DEBUG] [data.database] Disconnecting from database. exportactionlogsworker stdout | 2025-11-04 09:23:38,489 [65] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:38 GMT)" executed successfully pullstatsredisflushworker stdout | 2025-11-04 09:23:38,642 [79] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'pullstatsredisflushworker.py', 'pid': '79'} pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open pullstatsredisflushworker stdout | h.request(req.get_method(), req.selector, req.data, headers, pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request pullstatsredisflushworker stdout | self._send_request(method, url, body, headers, encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request pullstatsredisflushworker stdout | self.endheaders(body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders pullstatsredisflushworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output pullstatsredisflushworker stdout | self.send(msg) pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send pullstatsredisflushworker stdout | self.connect() pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect pullstatsredisflushworker stdout | self.sock = self._create_connection( pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection pullstatsredisflushworker stdout | raise exceptions[0] pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection pullstatsredisflushworker stdout | sock.connect(sa) pullstatsredisflushworker stdout | ConnectionRefusedError: [Errno 111] Connection refused pullstatsredisflushworker stdout | During handling of the above exception, another exception occurred: pullstatsredisflushworker stdout | Traceback (most recent call last): pullstatsredisflushworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run pullstatsredisflushworker stdout | push_to_gateway( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway pullstatsredisflushworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway pullstatsredisflushworker stdout | handler( pullstatsredisflushworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle pullstatsredisflushworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open pullstatsredisflushworker stdout | response = self._open(req, data) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open pullstatsredisflushworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain pullstatsredisflushworker stdout | result = func(*args) pullstatsredisflushworker stdout | ^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open pullstatsredisflushworker stdout | return self.do_open(http.client.HTTPConnection, req) pullstatsredisflushworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pullstatsredisflushworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open pullstatsredisflushworker stdout | raise URLError(err) pullstatsredisflushworker stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:39,676 [78] [DEBUG] [apscheduler.scheduler] Looking for jobs to run proxycacheblobworker stdout | 2025-11-04 09:23:39,676 [78] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:23:47.679968+00:00 (in 8.003229 seconds) proxycacheblobworker stdout | 2025-11-04 09:23:39,676 [78] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:49 GMT)" (scheduled at 2025-11-04 09:23:39.676299+00:00) proxycacheblobworker stdout | 2025-11-04 09:23:39,676 [78] [DEBUG] [workers.queueworker] Getting work item from queue. proxycacheblobworker stdout | 2025-11-04 09:23:39,677 [78] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 39, 677066), True, datetime.datetime(2025, 11, 4, 9, 23, 39, 677066), 0, 'proxycacheblob/%', 50, 1, 0]) proxycacheblobworker stdout | 2025-11-04 09:23:39,684 [78] [DEBUG] [workers.queueworker] No more work. proxycacheblobworker stdout | 2025-11-04 09:23:39,684 [78] [DEBUG] [data.database] Disconnecting from database. proxycacheblobworker stdout | 2025-11-04 09:23:39,684 [78] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:00:10], next run at: 2025-11-04 09:23:49 GMT)" executed successfully manifestbackfillworker stdout | 2025-11-04 09:23:39,760 [72] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'manifestbackfillworker.py', 'pid': '72'} manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open manifestbackfillworker stdout | h.request(req.get_method(), req.selector, req.data, headers, manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request manifestbackfillworker stdout | self._send_request(method, url, body, headers, encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request manifestbackfillworker stdout | self.endheaders(body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders manifestbackfillworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output manifestbackfillworker stdout | self.send(msg) manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send manifestbackfillworker stdout | self.connect() manifestbackfillworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect manifestbackfillworker stdout | self.sock = self._create_connection( manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection manifestbackfillworker stdout | raise exceptions[0] manifestbackfillworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection manifestbackfillworker stdout | sock.connect(sa) manifestbackfillworker stdout | ConnectionRefusedError: [Errno 111] Connection refused manifestbackfillworker stdout | During handling of the above exception, another exception occurred: manifestbackfillworker stdout | Traceback (most recent call last): manifestbackfillworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run manifestbackfillworker stdout | push_to_gateway( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway manifestbackfillworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway manifestbackfillworker stdout | handler( manifestbackfillworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle manifestbackfillworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open manifestbackfillworker stdout | response = self._open(req, data) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open manifestbackfillworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain manifestbackfillworker stdout | result = func(*args) manifestbackfillworker stdout | ^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open manifestbackfillworker stdout | return self.do_open(http.client.HTTPConnection, req) manifestbackfillworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ manifestbackfillworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open manifestbackfillworker stdout | raise URLError(err) manifestbackfillworker stdout | urllib.error.URLError: exportactionlogsworker stdout | 2025-11-04 09:23:40,026 [65] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'exportactionlogsworker.py', 'pid': '65'} exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open exportactionlogsworker stdout | h.request(req.get_method(), req.selector, req.data, headers, exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request exportactionlogsworker stdout | self._send_request(method, url, body, headers, encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request exportactionlogsworker stdout | self.endheaders(body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders exportactionlogsworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output exportactionlogsworker stdout | self.send(msg) exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send exportactionlogsworker stdout | self.connect() exportactionlogsworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect exportactionlogsworker stdout | self.sock = self._create_connection( exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection exportactionlogsworker stdout | raise exceptions[0] exportactionlogsworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection exportactionlogsworker stdout | sock.connect(sa) exportactionlogsworker stdout | ConnectionRefusedError: [Errno 111] Connection refused exportactionlogsworker stdout | During handling of the above exception, another exception occurred: exportactionlogsworker stdout | Traceback (most recent call last): exportactionlogsworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run exportactionlogsworker stdout | push_to_gateway( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway exportactionlogsworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway exportactionlogsworker stdout | handler( exportactionlogsworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle exportactionlogsworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open exportactionlogsworker stdout | response = self._open(req, data) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open exportactionlogsworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain exportactionlogsworker stdout | result = func(*args) exportactionlogsworker stdout | ^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open exportactionlogsworker stdout | return self.do_open(http.client.HTTPConnection, req) exportactionlogsworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exportactionlogsworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open exportactionlogsworker stdout | raise URLError(err) exportactionlogsworker stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:40,065 [254] [DEBUG] [app] Starting request: urn:request:ff7a985d-e6c7-4043-a3f5-afdb0ef7ae9a (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-web stdout | 2025-11-04 09:23:40,066 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-registry stdout | 2025-11-04 09:23:40,068 [250] [DEBUG] [app] Starting request: urn:request:6607e3ca-4344-4de9-9f1f-03b2a0cf5adb (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:40,068 [250] [DEBUG] [app] Ending request: urn:request:6607e3ca-4344-4de9-9f1f-03b2a0cf5adb (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:6607e3ca-4344-4de9-9f1f-03b2a0cf5adb', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 162 0.002) gunicorn-registry stdout | 2025-11-04 09:23:40,069 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:40,069 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:40,070 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:40,072 [261] [DEBUG] [app] Starting request: urn:request:f69d7607-7e01-4d7e-818e-a60175558a4d (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:40,072 [261] [DEBUG] [app] Ending request: urn:request:f69d7607-7e01-4d7e-818e-a60175558a4d (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:f69d7607-7e01-4d7e-818e-a60175558a4d', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:40,073 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:40,073 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.002 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:40,073 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:40,073 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:40,073 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:40,077 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:40,077 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:40,082 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:40,083 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:40,084 [254] [DEBUG] [app] Ending request: urn:request:ff7a985d-e6c7-4043-a3f5-afdb0ef7ae9a (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:ff7a985d-e6c7-4043-a3f5-afdb0ef7ae9a', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:40,084 [254] [DEBUG] [data.database] Disconnecting from database. gunicorn-web stdout | 2025-11-04 09:23:40,084 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.020 118 0.020) gunicorn-web stdout | 2025-11-04 09:23:40,123 [254] [DEBUG] [app] Starting request: urn:request:eb05ded5-da51-4a88-9218-0e1b904d7fe2 (/health/instance) {'X-Forwarded-For': '10.131.2.2'} gunicorn-registry stdout | 2025-11-04 09:23:40,125 [250] [DEBUG] [app] Starting request: urn:request:e9cb12f5-02cc-4de2-802a-3dc2f8179dc0 (/v1/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-registry stdout | 2025-11-04 09:23:40,125 [250] [DEBUG] [app] Ending request: urn:request:e9cb12f5-02cc-4de2-802a-3dc2f8179dc0 (/v1/_internal_ping) {'endpoint': 'v1.internal_ping', 'request_id': 'urn:request:e9cb12f5-02cc-4de2-802a-3dc2f8179dc0', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/v1/_internal_ping', 'path': '/v1/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '4a98b482', 'user-agent': 'python-requests/2.32.3'} gunicorn-registry stdout | 2025-11-04 09:23:40,125 [250] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" gunicorn-web stdout | 2025-11-04 09:23:40,125 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /v1/_internal_ping HTTP/1.1" 200 4 nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /v1/_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 162 0.001) gunicorn-web stdout | 2025-11-04 09:23:40,126 [254] [DEBUG] [urllib3.connectionpool] Resetting dropped connection: localhost gunicorn-web stdout | 2025-11-04 09:23:40,128 [261] [DEBUG] [app] Starting request: urn:request:dc6568a9-d72a-40a7-8352-940ab2f46085 (/_internal_ping) {'X-Forwarded-For': '127.0.0.1'} gunicorn-web stdout | 2025-11-04 09:23:40,128 [261] [DEBUG] [app] Ending request: urn:request:dc6568a9-d72a-40a7-8352-940ab2f46085 (/_internal_ping) {'endpoint': 'web.internal_ping', 'request_id': 'urn:request:dc6568a9-d72a-40a7-8352-940ab2f46085', 'remote_addr': '127.0.0.1', 'http_method': 'GET', 'original_url': 'http://localhost/_internal_ping', 'path': '/_internal_ping', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'python-requests/2.32.3'} gunicorn-web stdout | 2025-11-04 09:23:40,128 [261] [INFO] [gunicorn.access] 127.0.0.1 - - [04/Nov/2025:09:23:40 +0000] "GET /_internal_ping HTTP/1.0" 200 4 "-" "python-requests/2.32.3" nginx stdout | 127.0.0.1 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /_internal_ping HTTP/1.1" 200 4 "-" "python-requests/2.32.3" (0.001 159 0.001) gunicorn-web stdout | 2025-11-04 09:23:40,128 [254] [DEBUG] [urllib3.connectionpool] http://localhost:8080 "GET /_internal_ping HTTP/1.1" 200 4 gunicorn-web stdout | 2025-11-04 09:23:40,129 [254] [DEBUG] [data.model.health] Validating database connection. gunicorn-web stdout | 2025-11-04 09:23:40,129 [254] [INFO] [data.database] Setting database host to quayregistry-quay-database for worker gunicorn-web gunicorn-web stdout | 2025-11-04 09:23:40,129 [254] [INFO] [data.database] Connection pooling disabled for postgresql gunicorn-web stdout | 2025-11-04 09:23:40,132 [254] [DEBUG] [data.model.health] Checking for existence of team roles, timeout 5000 ms. gunicorn-web stdout | 2025-11-04 09:23:40,132 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (5000,)) gunicorn-web stdout | 2025-11-04 09:23:40,136 [254] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."name" FROM "teamrole" AS "t1" LIMIT %s', [1]) gunicorn-web stdout | 2025-11-04 09:23:40,137 [254] [DEBUG] [peewee] ('SET statement_timeout=%s;', (0,)) gunicorn-web stdout | 2025-11-04 09:23:40,138 [254] [DEBUG] [app] Ending request: urn:request:eb05ded5-da51-4a88-9218-0e1b904d7fe2 (/health/instance) {'endpoint': 'web.instance_health', 'request_id': 'urn:request:eb05ded5-da51-4a88-9218-0e1b904d7fe2', 'remote_addr': '10.131.2.2', 'http_method': 'GET', 'original_url': 'http://10.131.2.29/health/instance', 'path': '/health/instance', 'parameters': {}, 'json_body': None, 'confsha': '1f8eaf3d', 'user-agent': 'kube-probe/1.32'} gunicorn-web stdout | 2025-11-04 09:23:40,138 [254] [DEBUG] [data.database] Disconnecting from database. nginx stdout | 10.131.2.2 (-) - - [04/Nov/2025:09:23:40 +0000] "GET /health/instance HTTP/1.1" 200 152 "-" "kube-probe/1.32" (0.016 118 0.016) gunicorn-web stdout | 2025-11-04 09:23:40,139 [254] [INFO] [gunicorn.access] 10.131.2.2 - - [04/Nov/2025:09:23:40 +0000] "GET /health/instance HTTP/1.0" 200 152 "-" "kube-probe/1.32" repositorygcworker stdout | 2025-11-04 09:23:40,180 [90] [DEBUG] [apscheduler.scheduler] Looking for jobs to run repositorygcworker stdout | 2025-11-04 09:23:40,181 [90] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:06.265965+00:00 (in 26.084823 seconds) repositorygcworker stdout | 2025-11-04 09:23:40,181 [90] [INFO] [apscheduler.executors.default] Running job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:40 GMT)" (scheduled at 2025-11-04 09:23:40.180734+00:00) repositorygcworker stdout | 2025-11-04 09:23:40,181 [90] [DEBUG] [workers.queueworker] Getting work item from queue. repositorygcworker stdout | 2025-11-04 09:23:40,182 [90] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."queue_name", "t1"."body", "t1"."available_after", "t1"."available", "t1"."processing_expires", "t1"."retries_remaining", "t1"."state_id" FROM "queueitem" AS "t1" INNER JOIN (SELECT "t1"."id" FROM "queueitem" AS "t1" WHERE (((("t1"."available_after" <= %s) AND (("t1"."available" = %s) OR ("t1"."processing_expires" <= %s))) AND ("t1"."retries_remaining" > %s)) AND ("t1"."queue_name" ILIKE %s)) LIMIT %s) AS "j1" ON ("t1"."id" = "j1"."id") ORDER BY Random() LIMIT %s OFFSET %s', [datetime.datetime(2025, 11, 4, 9, 23, 40, 181503), True, datetime.datetime(2025, 11, 4, 9, 23, 40, 181503), 0, 'repositorygc/%', 50, 1, 0]) repositorygcworker stdout | 2025-11-04 09:23:40,188 [90] [DEBUG] [workers.queueworker] No more work. repositorygcworker stdout | 2025-11-04 09:23:40,188 [90] [DEBUG] [data.database] Disconnecting from database. repositorygcworker stdout | 2025-11-04 09:23:40,188 [90] [INFO] [apscheduler.executors.default] Job "QueueWorker.poll_queue (trigger: interval[0:01:00], next run at: 2025-11-04 09:24:40 GMT)" executed successfully securityworker stdout | 2025-11-04 09:23:40,437 [92] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'securityworker.py', 'pid': '92'} securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open securityworker stdout | h.request(req.get_method(), req.selector, req.data, headers, securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request securityworker stdout | self._send_request(method, url, body, headers, encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request securityworker stdout | self.endheaders(body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders securityworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output securityworker stdout | self.send(msg) securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send securityworker stdout | self.connect() securityworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect securityworker stdout | self.sock = self._create_connection( securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection securityworker stdout | raise exceptions[0] securityworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection securityworker stdout | sock.connect(sa) securityworker stdout | ConnectionRefusedError: [Errno 111] Connection refused securityworker stdout | During handling of the above exception, another exception occurred: securityworker stdout | Traceback (most recent call last): securityworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run securityworker stdout | push_to_gateway( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway securityworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway securityworker stdout | handler( securityworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle securityworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open securityworker stdout | response = self._open(req, data) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open securityworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain securityworker stdout | result = func(*args) securityworker stdout | ^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open securityworker stdout | return self.do_open(http.client.HTTPConnection, req) securityworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ securityworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open securityworker stdout | raise URLError(err) securityworker stdout | urllib.error.URLError: logrotateworker stdout | 2025-11-04 09:23:40,464 [71] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'logrotateworker.py', 'pid': '71'} logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open logrotateworker stdout | h.request(req.get_method(), req.selector, req.data, headers, logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request logrotateworker stdout | self._send_request(method, url, body, headers, encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request logrotateworker stdout | self.endheaders(body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders logrotateworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output logrotateworker stdout | self.send(msg) logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send logrotateworker stdout | self.connect() logrotateworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect logrotateworker stdout | self.sock = self._create_connection( logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection logrotateworker stdout | raise exceptions[0] logrotateworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection logrotateworker stdout | sock.connect(sa) logrotateworker stdout | ConnectionRefusedError: [Errno 111] Connection refused logrotateworker stdout | During handling of the above exception, another exception occurred: logrotateworker stdout | Traceback (most recent call last): logrotateworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run logrotateworker stdout | push_to_gateway( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway logrotateworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway logrotateworker stdout | handler( logrotateworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle logrotateworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open logrotateworker stdout | response = self._open(req, data) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open logrotateworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain logrotateworker stdout | result = func(*args) logrotateworker stdout | ^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open logrotateworker stdout | return self.do_open(http.client.HTTPConnection, req) logrotateworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ logrotateworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open logrotateworker stdout | raise URLError(err) logrotateworker stdout | urllib.error.URLError: quotaregistrysizeworker stdout | 2025-11-04 09:23:41,254 [82] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'quotaregistrysizeworker.py', 'pid': '82'} quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open quotaregistrysizeworker stdout | h.request(req.get_method(), req.selector, req.data, headers, quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request quotaregistrysizeworker stdout | self._send_request(method, url, body, headers, encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request quotaregistrysizeworker stdout | self.endheaders(body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders quotaregistrysizeworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output quotaregistrysizeworker stdout | self.send(msg) quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send quotaregistrysizeworker stdout | self.connect() quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect quotaregistrysizeworker stdout | self.sock = self._create_connection( quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection quotaregistrysizeworker stdout | raise exceptions[0] quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection quotaregistrysizeworker stdout | sock.connect(sa) quotaregistrysizeworker stdout | ConnectionRefusedError: [Errno 111] Connection refused quotaregistrysizeworker stdout | During handling of the above exception, another exception occurred: quotaregistrysizeworker stdout | Traceback (most recent call last): quotaregistrysizeworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run quotaregistrysizeworker stdout | push_to_gateway( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway quotaregistrysizeworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway quotaregistrysizeworker stdout | handler( quotaregistrysizeworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle quotaregistrysizeworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open quotaregistrysizeworker stdout | response = self._open(req, data) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open quotaregistrysizeworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain quotaregistrysizeworker stdout | result = func(*args) quotaregistrysizeworker stdout | ^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open quotaregistrysizeworker stdout | return self.do_open(http.client.HTTPConnection, req) quotaregistrysizeworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ quotaregistrysizeworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open quotaregistrysizeworker stdout | raise URLError(err) quotaregistrysizeworker stdout | urllib.error.URLError: blobuploadcleanupworker stdout | 2025-11-04 09:23:41,538 [59] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'blobuploadcleanupworker.py', 'pid': '59'} blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open blobuploadcleanupworker stdout | h.request(req.get_method(), req.selector, req.data, headers, blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request blobuploadcleanupworker stdout | self._send_request(method, url, body, headers, encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request blobuploadcleanupworker stdout | self.endheaders(body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders blobuploadcleanupworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output blobuploadcleanupworker stdout | self.send(msg) blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send blobuploadcleanupworker stdout | self.connect() blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect blobuploadcleanupworker stdout | self.sock = self._create_connection( blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection blobuploadcleanupworker stdout | raise exceptions[0] blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection blobuploadcleanupworker stdout | sock.connect(sa) blobuploadcleanupworker stdout | ConnectionRefusedError: [Errno 111] Connection refused blobuploadcleanupworker stdout | During handling of the above exception, another exception occurred: blobuploadcleanupworker stdout | Traceback (most recent call last): blobuploadcleanupworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run blobuploadcleanupworker stdout | push_to_gateway( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway blobuploadcleanupworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway blobuploadcleanupworker stdout | handler( blobuploadcleanupworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle blobuploadcleanupworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open blobuploadcleanupworker stdout | response = self._open(req, data) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open blobuploadcleanupworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain blobuploadcleanupworker stdout | result = func(*args) blobuploadcleanupworker stdout | ^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open blobuploadcleanupworker stdout | return self.do_open(http.client.HTTPConnection, req) blobuploadcleanupworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ blobuploadcleanupworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open blobuploadcleanupworker stdout | raise URLError(err) blobuploadcleanupworker stdout | urllib.error.URLError: autopruneworker stdout | 2025-11-04 09:23:41,838 [58] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'autopruneworker.py', 'pid': '58'} autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open autopruneworker stdout | h.request(req.get_method(), req.selector, req.data, headers, autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request autopruneworker stdout | self._send_request(method, url, body, headers, encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request autopruneworker stdout | self.endheaders(body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders autopruneworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output autopruneworker stdout | self.send(msg) autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send autopruneworker stdout | self.connect() autopruneworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect autopruneworker stdout | self.sock = self._create_connection( autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection autopruneworker stdout | raise exceptions[0] autopruneworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection autopruneworker stdout | sock.connect(sa) autopruneworker stdout | ConnectionRefusedError: [Errno 111] Connection refused autopruneworker stdout | During handling of the above exception, another exception occurred: autopruneworker stdout | Traceback (most recent call last): autopruneworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run autopruneworker stdout | push_to_gateway( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway autopruneworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway autopruneworker stdout | handler( autopruneworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle autopruneworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open autopruneworker stdout | response = self._open(req, data) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open autopruneworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain autopruneworker stdout | result = func(*args) autopruneworker stdout | ^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open autopruneworker stdout | return self.do_open(http.client.HTTPConnection, req) autopruneworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ autopruneworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open autopruneworker stdout | raise URLError(err) autopruneworker stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:42,546 [239] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '239'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:42,564 [69] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '69'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: gunicorn-secscan stdout | 2025-11-04 09:23:42,570 [238] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'secscan:application', 'pid': '238'} gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-secscan stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-secscan stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-secscan stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-secscan stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-secscan stdout | self.send(msg) gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-secscan stdout | self.connect() gunicorn-secscan stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-secscan stdout | self.sock = self._create_connection( gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-secscan stdout | sock.connect(sa) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-secscan stdout | self._internal_connect(address) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-secscan stdout | raise _SocketError(err, strerror(err)) gunicorn-secscan stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-secscan stdout | During handling of the above exception, another exception occurred: gunicorn-secscan stdout | Traceback (most recent call last): gunicorn-secscan stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-secscan stdout | push_to_gateway( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-secscan stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-secscan stdout | handler( gunicorn-secscan stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-secscan stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-secscan stdout | response = self._open(req, data) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-secscan stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-secscan stdout | result = func(*args) gunicorn-secscan stdout | ^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-secscan stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-secscan stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-secscan stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-secscan stdout | raise URLError(err) gunicorn-secscan stdout | urllib.error.URLError: servicekey stdout | 2025-11-04 09:23:42,952 [95] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'servicekeyworker.py', 'pid': '95'} servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open servicekey stdout | h.request(req.get_method(), req.selector, req.data, headers, servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request servicekey stdout | self._send_request(method, url, body, headers, encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request servicekey stdout | self.endheaders(body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders servicekey stdout | self._send_output(message_body, encode_chunked=encode_chunked) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output servicekey stdout | self.send(msg) servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send servicekey stdout | self.connect() servicekey stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect servicekey stdout | self.sock = self._create_connection( servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection servicekey stdout | raise exceptions[0] servicekey stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection servicekey stdout | sock.connect(sa) servicekey stdout | ConnectionRefusedError: [Errno 111] Connection refused servicekey stdout | During handling of the above exception, another exception occurred: servicekey stdout | Traceback (most recent call last): servicekey stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run servicekey stdout | push_to_gateway( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway servicekey stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway servicekey stdout | handler( servicekey stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle servicekey stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open servicekey stdout | response = self._open(req, data) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open servicekey stdout | result = self._call_chain(self.handle_open, protocol, protocol + servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain servicekey stdout | result = func(*args) servicekey stdout | ^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open servicekey stdout | return self.do_open(http.client.HTTPConnection, req) servicekey stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ servicekey stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open servicekey stdout | raise URLError(err) servicekey stdout | urllib.error.URLError: proxycacheblobworker stdout | 2025-11-04 09:23:43,357 [78] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'proxycacheblobworker.py', 'pid': '78'} proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open proxycacheblobworker stdout | h.request(req.get_method(), req.selector, req.data, headers, proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request proxycacheblobworker stdout | self._send_request(method, url, body, headers, encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request proxycacheblobworker stdout | self.endheaders(body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders proxycacheblobworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output proxycacheblobworker stdout | self.send(msg) proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send proxycacheblobworker stdout | self.connect() proxycacheblobworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect proxycacheblobworker stdout | self.sock = self._create_connection( proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection proxycacheblobworker stdout | raise exceptions[0] proxycacheblobworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection proxycacheblobworker stdout | sock.connect(sa) proxycacheblobworker stdout | ConnectionRefusedError: [Errno 111] Connection refused proxycacheblobworker stdout | During handling of the above exception, another exception occurred: proxycacheblobworker stdout | Traceback (most recent call last): proxycacheblobworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run proxycacheblobworker stdout | push_to_gateway( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway proxycacheblobworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway proxycacheblobworker stdout | handler( proxycacheblobworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle proxycacheblobworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open proxycacheblobworker stdout | response = self._open(req, data) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open proxycacheblobworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain proxycacheblobworker stdout | result = func(*args) proxycacheblobworker stdout | ^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open proxycacheblobworker stdout | return self.do_open(http.client.HTTPConnection, req) proxycacheblobworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ proxycacheblobworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open proxycacheblobworker stdout | raise URLError(err) proxycacheblobworker stdout | urllib.error.URLError: gcworker stdout | 2025-11-04 09:23:43,740 [66] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'gcworker.py', 'pid': '66'} gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gcworker stdout | self._send_request(method, url, body, headers, encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gcworker stdout | self.send(msg) gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gcworker stdout | self.connect() gcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gcworker stdout | self.sock = self._create_connection( gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection gcworker stdout | raise exceptions[0] gcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection gcworker stdout | sock.connect(sa) gcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused gcworker stdout | During handling of the above exception, another exception occurred: gcworker stdout | Traceback (most recent call last): gcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gcworker stdout | push_to_gateway( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gcworker stdout | handler( gcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gcworker stdout | response = self._open(req, data) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gcworker stdout | result = func(*args) gcworker stdout | ^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gcworker stdout | return self.do_open(http.client.HTTPConnection, req) gcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gcworker stdout | raise URLError(err) gcworker stdout | urllib.error.URLError: namespacegcworker stdout | 2025-11-04 09:23:44,566 [75] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'namespacegcworker.py', 'pid': '75'} namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open namespacegcworker stdout | h.request(req.get_method(), req.selector, req.data, headers, namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request namespacegcworker stdout | self._send_request(method, url, body, headers, encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request namespacegcworker stdout | self.endheaders(body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders namespacegcworker stdout | self._send_output(message_body, encode_chunked=encode_chunked) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output namespacegcworker stdout | self.send(msg) namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send namespacegcworker stdout | self.connect() namespacegcworker stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect namespacegcworker stdout | self.sock = self._create_connection( namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 865, in create_connection namespacegcworker stdout | raise exceptions[0] namespacegcworker stdout | File "/usr/lib64/python3.12/socket.py", line 850, in create_connection namespacegcworker stdout | sock.connect(sa) namespacegcworker stdout | ConnectionRefusedError: [Errno 111] Connection refused namespacegcworker stdout | During handling of the above exception, another exception occurred: namespacegcworker stdout | Traceback (most recent call last): namespacegcworker stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run namespacegcworker stdout | push_to_gateway( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway namespacegcworker stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway namespacegcworker stdout | handler( namespacegcworker stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle namespacegcworker stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open namespacegcworker stdout | response = self._open(req, data) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open namespacegcworker stdout | result = self._call_chain(self.handle_open, protocol, protocol + namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain namespacegcworker stdout | result = func(*args) namespacegcworker stdout | ^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open namespacegcworker stdout | return self.do_open(http.client.HTTPConnection, req) namespacegcworker stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ namespacegcworker stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open namespacegcworker stdout | raise URLError(err) namespacegcworker stdout | urllib.error.URLError: buildlogsarchiver stdout | 2025-11-04 09:23:44,980 [61] [DEBUG] [apscheduler.scheduler] Looking for jobs to run buildlogsarchiver stdout | 2025-11-04 09:23:44,980 [61] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2025-11-04 09:24:14.980431+00:00 (in 29.999510 seconds) buildlogsarchiver stdout | 2025-11-04 09:23:44,981 [61] [INFO] [apscheduler.executors.default] Running job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:14 GMT)" (scheduled at 2025-11-04 09:23:44.980431+00:00) buildlogsarchiver stdout | 2025-11-04 09:23:44,982 [61] [DEBUG] [peewee] ('SELECT "candidates"."id" FROM (SELECT "t1"."id" FROM "repositorybuild" AS "t1" WHERE ((("t1"."phase" IN (%s, %s, %s)) OR ("t1"."started" < %s)) AND ("t1"."logs_archived" = %s)) LIMIT %s) AS "candidates" ORDER BY Random() LIMIT %s OFFSET %s', ['complete', 'error', 'cancelled', datetime.datetime(2025, 10, 20, 9, 23, 44, 981264), False, 50, 1, 0]) buildlogsarchiver stdout | 2025-11-04 09:23:44,990 [61] [DEBUG] [__main__] No more builds to archive buildlogsarchiver stdout | 2025-11-04 09:23:44,990 [61] [DEBUG] [data.database] Disconnecting from database. buildlogsarchiver stdout | 2025-11-04 09:23:44,990 [61] [INFO] [apscheduler.executors.default] Job "ArchiveBuildLogsWorker._archive_redis_buildlogs (trigger: interval[0:00:30], next run at: 2025-11-04 09:24:14 GMT)" executed successfully gunicorn-web stdout | 2025-11-04 09:23:45,058 [254] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '254'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:45,081 [257] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '257'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:45,091 [258] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '258'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:45,099 [70] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '70'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-web stdout | 2025-11-04 09:23:45,224 [261] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'web:application', 'pid': '261'} gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-web stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-web stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-web stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-web stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-web stdout | self.send(msg) gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-web stdout | self.connect() gunicorn-web stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-web stdout | self.sock = self._create_connection( gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-web stdout | sock.connect(sa) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-web stdout | self._internal_connect(address) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-web stdout | raise _SocketError(err, strerror(err)) gunicorn-web stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-web stdout | During handling of the above exception, another exception occurred: gunicorn-web stdout | Traceback (most recent call last): gunicorn-web stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-web stdout | push_to_gateway( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-web stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-web stdout | handler( gunicorn-web stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-web stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-web stdout | response = self._open(req, data) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-web stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-web stdout | result = func(*args) gunicorn-web stdout | ^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-web stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-web stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-web stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-web stdout | raise URLError(err) gunicorn-web stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,599 [255] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '255'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,659 [250] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '250'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,696 [252] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '252'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,743 [251] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '251'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,748 [256] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '256'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,759 [253] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '253'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,775 [259] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '259'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,784 [260] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '260'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: gunicorn-registry stdout | 2025-11-04 09:23:46,768 [68] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quayregistry-quay-app-6f599c49b4-v4s6b', 'process_name': 'registry:application', 'pid': '68'} gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1344, in do_open gunicorn-registry stdout | h.request(req.get_method(), req.selector, req.data, headers, gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1338, in request gunicorn-registry stdout | self._send_request(method, url, body, headers, encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1384, in _send_request gunicorn-registry stdout | self.endheaders(body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1333, in endheaders gunicorn-registry stdout | self._send_output(message_body, encode_chunked=encode_chunked) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1093, in _send_output gunicorn-registry stdout | self.send(msg) gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1037, in send gunicorn-registry stdout | self.connect() gunicorn-registry stdout | File "/usr/lib64/python3.12/http/client.py", line 1003, in connect gunicorn-registry stdout | self.sock = self._create_connection( gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/socket.py", line 115, in create_connection gunicorn-registry stdout | sock.connect(sa) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 586, in connect gunicorn-registry stdout | self._internal_connect(address) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/gevent/_socketcommon.py", line 630, in _internal_connect gunicorn-registry stdout | raise _SocketError(err, strerror(err)) gunicorn-registry stdout | ConnectionRefusedError: [Errno 111] Connection refused gunicorn-registry stdout | During handling of the above exception, another exception occurred: gunicorn-registry stdout | Traceback (most recent call last): gunicorn-registry stdout | File "/quay-registry/util/metrics/prometheus.py", line 140, in run gunicorn-registry stdout | push_to_gateway( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway gunicorn-registry stdout | _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler) gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway gunicorn-registry stdout | handler( gunicorn-registry stdout | File "/opt/app-root/lib64/python3.12/site-packages/prometheus_client/exposition.py", line 221, in handle gunicorn-registry stdout | resp = build_opener(HTTPHandler).open(request, timeout=timeout) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 515, in open gunicorn-registry stdout | response = self._open(req, data) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 532, in _open gunicorn-registry stdout | result = self._call_chain(self.handle_open, protocol, protocol + gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 492, in _call_chain gunicorn-registry stdout | result = func(*args) gunicorn-registry stdout | ^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1373, in http_open gunicorn-registry stdout | return self.do_open(http.client.HTTPConnection, req) gunicorn-registry stdout | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ gunicorn-registry stdout | File "/usr/lib64/python3.12/urllib/request.py", line 1347, in do_open gunicorn-registry stdout | raise URLError(err) gunicorn-registry stdout | urllib.error.URLError: